From 6ed7ee7409d56183ed84a3e1b562603d0846ff5b Mon Sep 17 00:00:00 2001 From: Meysam Hadeli <35596795+meysamhadeli@users.noreply.github.com> Date: Fri, 14 Feb 2025 01:13:01 +0330 Subject: [PATCH 1/3] feat: add full observability top of open telemetry collector --- deployments/configs/dashboards.md | 8 + .../dashboards/dotnet-core-endpoint.json} | 0 .../grafana/dashboards/dotnet-core.json} | 0 .../grafana/dashboards/node-exporter.json | 892 ++ .../grafana/dashboards/postgresql.json | 1459 +++ .../configs/grafana/dashboards/rabbitmq.json | 8209 +++++++++++++++++ .../provisioning/dashboards/dashboard.yml | 14 + .../provisioning/datasources/datasource.yml | 88 + deployments/configs/loki-config.yaml | 44 + .../configs/otel-collector-config.yaml | 131 + deployments/configs/prometheus.yaml | 48 + deployments/configs/tempo.yaml | 49 + .../docker-compose.infrastructure.yaml | 362 + .../docker-compose/docker-compose.yaml | 378 +- .../docker-compose/infrastructure.yaml | 234 - .../monitoring/prom/prometheus.yml | 69 - .../kubernetes/booking-microservices.yml | 1311 ++- src/BuildingBlocks/BuildingBlocks.csproj | 28 +- src/BuildingBlocks/HealthCheck/Extensions.cs | 5 +- src/BuildingBlocks/Logging/ElasticOptions.cs | 8 - src/BuildingBlocks/Logging/Extensions.cs | 33 - src/BuildingBlocks/Logging/LogOptions.cs | 3 - src/BuildingBlocks/Logging/SentryOptions.cs | 9 - .../OpenTelemetry/Extensions.cs | 44 - .../ActivityExtensions.cs | 175 + .../OpenTelemetryCollector/ActivityInfo.cs | 29 + .../ObservabilityPipelineBehavior.cs | 85 + .../Commands/CommandHandlerActivity.cs | 86 + .../Commands/CommandHandlerMetrics.cs | 157 + .../Query/QueryHandlerActivity.cs | 47 + .../Query/QueryHandlerMetrics.cs | 156 + .../CreateActivityInfo.cs | 12 + .../CustomeDiagnosticsProvider.cs | 132 + .../IDiagnosticsProvider.cs | 23 + .../OpenTelemetryCollector/Extensions.cs | 358 + .../ObservabilityConstant.cs | 16 + .../ObservabilityOptions.cs | 47 + .../OpenTelemetryCollector/TelemetryTags.cs | 265 + .../src/Booking.Api/appsettings.docker.json | 10 - .../Booking/src/Booking.Api/appsettings.json | 137 +- .../InfrastructureExtensions.cs | 6 +- .../src/Flight.Api/appsettings.docker.json | 10 - .../Flight/src/Flight.Api/appsettings.json | 113 +- .../InfrastructureExtensions.cs | 6 +- .../Flight/Features/GetFlightByIdTests.cs | 2 - .../src/Identity.Api/appsettings.docker.json | 10 - .../src/Identity.Api/appsettings.json | 99 +- .../InfrastructureExtensions.cs | 6 +- .../src/Passenger.Api/appsettings.docker.json | 10 - .../src/Passenger.Api/appsettings.json | 113 +- .../InfrastructureExtensions.cs | 66 +- .../Features/GetPassengerByIdTests.cs | 2 - 52 files changed, 14113 insertions(+), 1491 deletions(-) create mode 100644 deployments/configs/dashboards.md rename deployments/{docker-compose/monitoring/dashboards/ASP.NET Core Endpoint.json => configs/grafana/dashboards/dotnet-core-endpoint.json} (100%) rename deployments/{docker-compose/monitoring/dashboards/ASP.NET Core.json => configs/grafana/dashboards/dotnet-core.json} (100%) create mode 100644 deployments/configs/grafana/dashboards/node-exporter.json create mode 100644 deployments/configs/grafana/dashboards/postgresql.json create mode 100644 deployments/configs/grafana/dashboards/rabbitmq.json create mode 100644 deployments/configs/grafana/provisioning/dashboards/dashboard.yml create mode 100644 deployments/configs/grafana/provisioning/datasources/datasource.yml create mode 100644 deployments/configs/loki-config.yaml create mode 100644 deployments/configs/otel-collector-config.yaml create mode 100644 deployments/configs/prometheus.yaml create mode 100644 deployments/configs/tempo.yaml create mode 100644 deployments/docker-compose/docker-compose.infrastructure.yaml delete mode 100644 deployments/docker-compose/infrastructure.yaml delete mode 100644 deployments/docker-compose/monitoring/prom/prometheus.yml delete mode 100644 src/BuildingBlocks/Logging/ElasticOptions.cs delete mode 100644 src/BuildingBlocks/Logging/SentryOptions.cs delete mode 100644 src/BuildingBlocks/OpenTelemetry/Extensions.cs create mode 100644 src/BuildingBlocks/OpenTelemetryCollector/ActivityExtensions.cs create mode 100644 src/BuildingBlocks/OpenTelemetryCollector/ActivityInfo.cs create mode 100644 src/BuildingBlocks/OpenTelemetryCollector/Behaviors/ObservabilityPipelineBehavior.cs create mode 100644 src/BuildingBlocks/OpenTelemetryCollector/CoreDiagnostics/Commands/CommandHandlerActivity.cs create mode 100644 src/BuildingBlocks/OpenTelemetryCollector/CoreDiagnostics/Commands/CommandHandlerMetrics.cs create mode 100644 src/BuildingBlocks/OpenTelemetryCollector/CoreDiagnostics/Query/QueryHandlerActivity.cs create mode 100644 src/BuildingBlocks/OpenTelemetryCollector/CoreDiagnostics/Query/QueryHandlerMetrics.cs create mode 100644 src/BuildingBlocks/OpenTelemetryCollector/CreateActivityInfo.cs create mode 100644 src/BuildingBlocks/OpenTelemetryCollector/DiagnosticsProvider/CustomeDiagnosticsProvider.cs create mode 100644 src/BuildingBlocks/OpenTelemetryCollector/DiagnosticsProvider/IDiagnosticsProvider.cs create mode 100644 src/BuildingBlocks/OpenTelemetryCollector/Extensions.cs create mode 100644 src/BuildingBlocks/OpenTelemetryCollector/ObservabilityConstant.cs create mode 100644 src/BuildingBlocks/OpenTelemetryCollector/ObservabilityOptions.cs create mode 100644 src/BuildingBlocks/OpenTelemetryCollector/TelemetryTags.cs diff --git a/deployments/configs/dashboards.md b/deployments/configs/dashboards.md new file mode 100644 index 00000000..999b1d8f --- /dev/null +++ b/deployments/configs/dashboards.md @@ -0,0 +1,8 @@ +# Dashboards + +- [Introducing ASP.NET Core metrics and Grafana dashboards in .NET 8](https://devblogs.microsoft.com/dotnet/introducing-aspnetcore-metrics-and-grafana-dashboards-in-dotnet-8/) +- [ASP.NET Core](https://grafana.com/grafana/dashboards/19924-asp-net-core/) +- [ASP.NET Core Endpoint](https://grafana.com/grafana/dashboards/19925-asp-net-core-endpoint/) +- [Node Exporter Quickstart and Dashboard](https://grafana.com/grafana/dashboards/13978-node-exporter-quickstart-and-dashboard/) +- [PostgreSQL Exporter Quickstart and Dashboard](https://grafana.com/grafana/dashboards/14114-postgres-overview/) +- [RabbitMQ-Overview](https://grafana.com/grafana/dashboards/10991-rabbitmq-overview/) \ No newline at end of file diff --git a/deployments/docker-compose/monitoring/dashboards/ASP.NET Core Endpoint.json b/deployments/configs/grafana/dashboards/dotnet-core-endpoint.json similarity index 100% rename from deployments/docker-compose/monitoring/dashboards/ASP.NET Core Endpoint.json rename to deployments/configs/grafana/dashboards/dotnet-core-endpoint.json diff --git a/deployments/docker-compose/monitoring/dashboards/ASP.NET Core.json b/deployments/configs/grafana/dashboards/dotnet-core.json similarity index 100% rename from deployments/docker-compose/monitoring/dashboards/ASP.NET Core.json rename to deployments/configs/grafana/dashboards/dotnet-core.json diff --git a/deployments/configs/grafana/dashboards/node-exporter.json b/deployments/configs/grafana/dashboards/node-exporter.json new file mode 100644 index 00000000..4839d46a --- /dev/null +++ b/deployments/configs/grafana/dashboards/node-exporter.json @@ -0,0 +1,892 @@ +{ + "__inputs": [], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "7.4.3" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } + ], + "annotations": { + "list": [] + }, + "editable": false, + "gnetId": 13978, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [], + "refresh": "", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": {}, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n (1 - rate(node_cpu_seconds_total{job=\"node\", mode=\"idle\", instance=\"$instance\"}[$__interval]))\n/ ignoring(cpu) group_left\n count without (cpu)( node_cpu_seconds_total{job=\"node\", mode=\"idle\", instance=\"$instance\"})\n)\n", + "format": "time_series", + "interval": "1m", + "intervalFactor": 5, + "legendFormat": "{{cpu}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": {}, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{job=\"node\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "1m load average", + "refId": "A" + }, + { + "expr": "node_load5{job=\"node\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5m load average", + "refId": "B" + }, + { + "expr": "node_load15{job=\"node\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "15m load average", + "refId": "C" + }, + { + "expr": "count(node_cpu_seconds_total{job=\"node\", instance=\"$instance\", mode=\"idle\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "logical cores", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": {}, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n node_memory_MemTotal_bytes{job=\"node\", instance=\"$instance\"}\n-\n node_memory_MemFree_bytes{job=\"node\", instance=\"$instance\"}\n-\n node_memory_Buffers_bytes{job=\"node\", instance=\"$instance\"}\n-\n node_memory_Cached_bytes{job=\"node\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory used", + "refId": "A" + }, + { + "expr": "node_memory_Buffers_bytes{job=\"node\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory buffers", + "refId": "B" + }, + { + "expr": "node_memory_Cached_bytes{job=\"node\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory cached", + "refId": "C" + }, + { + "expr": "node_memory_MemFree_bytes{job=\"node\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory free", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": {}, + "id": 5, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "100 -\n(\n avg(node_memory_MemAvailable_bytes{job=\"node\", instance=\"$instance\"})\n/\n avg(node_memory_MemTotal_bytes{job=\"node\", instance=\"$instance\"})\n* 100\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Memory Usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": {}, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ read| written/", + "yaxis": 1 + }, + { + "alias": "/ io time/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_read_bytes_total{job=\"node\", instance=\"$instance\", device!=\"\"}[$__interval])", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{device}} read", + "refId": "A" + }, + { + "expr": "rate(node_disk_written_bytes_total{job=\"node\", instance=\"$instance\", device!=\"\"}[$__interval])", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{device}} written", + "refId": "B" + }, + { + "expr": "rate(node_disk_io_time_seconds_total{job=\"node\", instance=\"$instance\", device!=\"\"}[$__interval])", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{device}} io time", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": {}, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "used", + "color": "#E0B400" + }, + { + "alias": "available", + "color": "#73BF69" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n max by (device) (\n node_filesystem_size_bytes{job=\"node\", instance=\"$instance\", fstype!=\"\"}\n -\n node_filesystem_avail_bytes{job=\"node\", instance=\"$instance\", fstype!=\"\"}\n )\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "used", + "refId": "A" + }, + { + "expr": "sum(\n max by (device) (\n node_filesystem_avail_bytes{job=\"node\", instance=\"$instance\", fstype!=\"\"}\n )\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "available", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": {}, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{job=\"node\", instance=\"$instance\", device!=\"lo\"}[$__interval])", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{device}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": {}, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_transmit_bytes_total{job=\"node\", instance=\"$instance\", device!=\"lo\"}[$__interval])", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{device}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network Transmitted", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(node_exporter_build_info{job=\"node\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Node Exporter Quickstart and Dashboard", + "version": 0, + "description": "A quickstart to setup Prometheus Node Exporter with preconfigured dashboards, alerting rules, and recording rules." +} \ No newline at end of file diff --git a/deployments/configs/grafana/dashboards/postgresql.json b/deployments/configs/grafana/dashboards/postgresql.json new file mode 100644 index 00000000..d1d696f4 --- /dev/null +++ b/deployments/configs/grafana/dashboards/postgresql.json @@ -0,0 +1,1459 @@ +{ + "__inputs": [], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "7.2.0" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A quickstart to setup the Prometheus PostgreSQL Exporter with preconfigured dashboards, alerting rules, and recording rules.", + "editable": true, + "gnetId": 14114, + "graphTooltip": 0, + "id": 1, + "iteration": 1603191461722, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 20, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 1, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "fetched", + "dsType": "prometheus", + "expr": "sum(irate(pg_stat_database_tup_fetched{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "fetched", + "measurement": "postgresql", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tup_fetched" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "10s" + ], + "type": "non_negative_derivative" + } + ] + ], + "step": 120, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + }, + { + "alias": "fetched", + "dsType": "prometheus", + "expr": "sum(irate(pg_stat_database_tup_returned{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "returned", + "measurement": "postgresql", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tup_fetched" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "10s" + ], + "type": "non_negative_derivative" + } + ] + ], + "step": 120, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + }, + { + "alias": "fetched", + "dsType": "prometheus", + "expr": "sum(irate(pg_stat_database_tup_inserted{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "inserted", + "measurement": "postgresql", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tup_fetched" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "10s" + ], + "type": "non_negative_derivative" + } + ] + ], + "step": 120, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + }, + { + "alias": "fetched", + "dsType": "prometheus", + "expr": "sum(irate(pg_stat_database_tup_updated{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "updated", + "measurement": "postgresql", + "policy": "default", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tup_fetched" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "10s" + ], + "type": "non_negative_derivative" + } + ] + ], + "step": 120, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + }, + { + "alias": "fetched", + "dsType": "prometheus", + "expr": "sum(irate(pg_stat_database_tup_deleted{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "deleted", + "measurement": "postgresql", + "policy": "default", + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tup_fetched" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "10s" + ], + "type": "non_negative_derivative" + } + ] + ], + "step": 120, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Rows", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "decimals": 0, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 0 + }, + "height": "55px", + "id": 11, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "dsType": "prometheus", + "expr": "sum(irate(pg_stat_database_xact_commit{datname=~\"$db\",instance=~\"$instance\"}[5m])) + sum(irate(pg_stat_database_xact_rollback{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "measurement": "postgresql", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "xact_commit" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "10s" + ], + "type": "non_negative_derivative" + } + ] + ], + "step": 1800, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + } + ], + "thresholds": "", + "title": "QPS", + "transparent": true, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 1, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 2, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Buffers Allocated", + "dsType": "prometheus", + "expr": "irate(pg_stat_bgwriter_buffers_alloc{instance=~'$instance'}[5m])", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "buffers_alloc", + "measurement": "postgresql", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "buffers_alloc" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "step": 240, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + }, + { + "alias": "Buffers Allocated", + "dsType": "prometheus", + "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync{instance=~'$instance'}[5m])", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "buffers_backend_fsync", + "measurement": "postgresql", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "buffers_alloc" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "step": 240, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + }, + { + "alias": "Buffers Allocated", + "dsType": "prometheus", + "expr": "irate(pg_stat_bgwriter_buffers_backend{instance=~'$instance'}[5m])", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "buffers_backend", + "measurement": "postgresql", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "buffers_alloc" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "step": 240, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + }, + { + "alias": "Buffers Allocated", + "dsType": "prometheus", + "expr": "irate(pg_stat_bgwriter_buffers_clean{instance=~'$instance'}[5m])", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "buffers_clean", + "measurement": "postgresql", + "policy": "default", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "buffers_alloc" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "step": 240, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + }, + { + "alias": "Buffers Allocated", + "dsType": "prometheus", + "expr": "irate(pg_stat_bgwriter_buffers_checkpoint{instance=~'$instance'}[5m])", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "buffers_checkpoint", + "measurement": "postgresql", + "policy": "default", + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "buffers_alloc" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "step": 240, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Buffers", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "id": 3, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "conflicts", + "dsType": "prometheus", + "expr": "sum(rate(pg_stat_database_deadlocks{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "deadlocks", + "measurement": "postgresql", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "conflicts" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "step": 240, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + }, + { + "alias": "deadlocks", + "dsType": "prometheus", + "expr": "sum(rate(pg_stat_database_conflicts{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "conflicts", + "measurement": "postgresql", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "deadlocks" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "step": 240, + "tags": [ + { + "key": "instance", + "operator": "=~", + "value": "/^$instance$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Conflicts/Deadlocks", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 12, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": true, + "pluginVersion": "7.2.1", + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(pg_stat_database_blks_hit{datname=~\"$db\",instance=~\"$instance\"}) / (sum(pg_stat_database_blks_hit{datname=~\"$db\",instance=~\"$instance\"}) + sum(pg_stat_database_blks_read{datname=~\"$db\",instance=~\"$instance\"}))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "cache hit rate", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cache hit ratio", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 14 + }, + "hiddenSeries": false, + "id": 13, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pg_stat_database_numbackends{datname=~\"$db\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{__name__}}", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of active connections", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [ + "postgres" + ], + "templating": { + "list": [ + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "definition": "", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(up{job=~\"postgres.*\"},instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "definition": "label_values(pg_stat_database_tup_fetched{instance=~\"$instance\",datname!~\"template.*|postgres\"},datname)", + "hide": 0, + "includeAll": true, + "label": "db", + "multi": false, + "name": "db", + "options": [], + "query": "label_values(pg_stat_database_tup_fetched{instance=~\"$instance\",datname!~\"template.*|postgres\"},datname)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "Postgres Overview", + "value": "Postgres Overview" + }, + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "postgres", + "value": "postgres" + }, + "datasource": "$datasource", + "definition": "label_values(pg_up, job)", + "hide": 0, + "includeAll": false, + "label": "job", + "multi": false, + "name": "job", + "options": [ + { + "selected": true, + "text": "postgres", + "value": "postgres" + } + ], + "query": "label_values(pg_up, job)", + "refresh": 0, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "PostgreSQL Exporter Quickstart and Dashboard", + "uid": "wGgaPlciz", + "version": 5 +} \ No newline at end of file diff --git a/deployments/configs/grafana/dashboards/rabbitmq.json b/deployments/configs/grafana/dashboards/rabbitmq.json new file mode 100644 index 00000000..b199b4b7 --- /dev/null +++ b/deployments/configs/grafana/dashboards/rabbitmq.json @@ -0,0 +1,8209 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": [], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "8.3.4" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "A new RabbitMQ Management Overview", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "iteration": 1659711638455, + "links": [ + { + "icon": "doc", + "tags": [], + "targetBlank": true, + "title": "Monitoring with Prometheus & Grafana", + "tooltip": "", + "type": "link", + "url": "https://www.rabbitmq.com/prometheus.html" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#37872D", + "value": null + }, + { + "color": "#1F60C4", + "value": 10000 + }, + { + "color": "#C4162A", + "value": 100000 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "id": 64, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rabbitmq_queue_messages_ready * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Ready messages", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#C4162A", + "value": null + }, + { + "color": "#1F60C4", + "value": -1 + }, + { + "color": "#37872D", + "value": 50 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 62, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_received_total[60s]) * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Incoming messages / s", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#C4162A", + "value": null + }, + { + "color": "#1F60C4", + "value": 0 + }, + { + "color": "#37872D", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 66, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rabbitmq_global_publishers * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Publishers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#C4162A", + "value": null + }, + { + "color": "#1F60C4", + "value": 0 + }, + { + "color": "#37872D", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 37, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rabbitmq_connections * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Connections", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#C4162A", + "value": null + }, + { + "color": "#1F60C4", + "value": 0 + }, + { + "color": "#37872D", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 40, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rabbitmq_queues * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Queues", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#37872D", + "value": null + }, + { + "color": "#1F60C4", + "value": 100 + }, + { + "color": "#C4162A", + "value": 500 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 3 + }, + "id": 65, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rabbitmq_queue_messages_unacked * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Unacknowledged messages", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#C4162A", + "value": null + }, + { + "color": "#1F60C4", + "value": -1 + }, + { + "color": "#37872D", + "value": 50 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 3 + }, + "id": 63, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_redelivered_total[60s]) * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_global_messages_delivered_consume_auto_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_global_messages_delivered_consume_manual_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_global_messages_delivered_get_auto_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_global_messages_delivered_get_manual_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Outgoing messages / s", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#C4162A", + "value": null + }, + { + "color": "#1F60C4", + "value": 0 + }, + { + "color": "#37872D", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 3 + }, + "id": 41, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rabbitmq_consumers * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Consumers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#C4162A", + "value": null + }, + { + "color": "#1F60C4", + "value": 0 + }, + { + "color": "#37872D", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 3 + }, + "id": 38, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rabbitmq_channels * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Channels", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#1F60C4", + "value": null + }, + { + "color": "#37872D", + "value": 3 + }, + { + "color": "#C4162A", + "value": 8 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 3 + }, + "id": 67, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rabbitmq_build_info * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Nodes", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 4, + "panels": [], + "title": "NODES", + "type": "row" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "erlang_version" + }, + "properties": [ + { + "id": "displayName", + "value": "Erlang/OTP" + }, + { + "id": "unit", + "value": "none" + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)" + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "rabbitmq_version" + }, + "properties": [ + { + "id": "displayName", + "value": "RabbitMQ" + }, + { + "id": "unit", + "value": "none" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)" + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "displayName", + "value": "Host" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "rabbitmq_node" + }, + "properties": [ + { + "id": "displayName", + "value": "Node name" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)" + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "rabbitmq_cluster" + }, + "properties": [ + { + "id": "displayName", + "value": "Cluster" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "prometheus_client_version" + }, + "properties": [ + { + "id": "displayName", + "value": "prometheus.erl" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "prometheus_plugin_version" + }, + "properties": [ + { + "id": "displayName", + "value": "rabbitmq_prometheus" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 69, + "links": [], + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "rabbitmq_build_info * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}", + "format": "table", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "transformations": [ + { + "id": "merge", + "options": { + "reducers": [] + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "If the value is zero or less, the memory alarm will be triggered and all publishing connections across all cluster nodes will be blocked.\n\nThis value can temporarily go negative because the memory alarm is triggered with a slight delay.\n\nThe kernel's view of the amount of memory used by the node can differ from what the node itself can observe. This means that this value can be negative for a sustained period of time.\n\nBy default nodes use resident set size (RSS) to compute how much memory they use. This strategy can be changed (see the guides below).\n\n* [Alarms](https://www.rabbitmq.com/alarms.html)\n* [Memory Alarms](https://www.rabbitmq.com/memory.html)\n* [Reasoning About Memory Use](https://www.rabbitmq.com/memory-use.html)\n* [Blocked Connection Notifications](https://www.rabbitmq.com/connection-blocked.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "decimals": 1, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 0 + }, + { + "color": "transparent", + "value": 536870912 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 7, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "(rabbitmq_resident_memory_limit_bytes * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) -\n(rabbitmq_process_resident_memory_bytes * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Memory available before publishers blocked", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric is reported for the partition where the RabbitMQ data directory is stored.\n\nIf the value is zero or less, the disk alarm will be triggered and all publishing connections across all cluster nodes will be blocked.\n\nThis value can temporarily go negative because the free disk space alarm is triggered with a slight delay.\n\n* [Alarms](https://www.rabbitmq.com/alarms.html)\n* [Disk Space Alarms](https://www.rabbitmq.com/disk-alarms.html)\n* [Disk Space](https://www.rabbitmq.com/production-checklist.html#resource-limits-disk-space)\n* [Persistence Configuration](https://www.rabbitmq.com/persistence-conf.html)\n* [Blocked Connection Notifications](https://www.rabbitmq.com/connection-blocked.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "decimals": 1, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 1073741824 + }, + { + "color": "transparent", + "value": 5368709120 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 12, + "y": 11 + }, + "id": 8, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "rabbitmq_disk_space_available_bytes * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Disk space available before publishers blocked", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "When this value reaches zero, new connections will not be accepted and disk write operations may fail.\n\nClient libraries, peer nodes and CLI tools will not be able to connect when the node runs out of available file descriptors.\n\n* [Open File Handles Limit](https://www.rabbitmq.com/production-checklist.html#resource-limits-file-handle-limit)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 500 + }, + { + "color": "transparent", + "value": 1000 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 11 + }, + "id": 2, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "(rabbitmq_process_max_fds * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) -\n(rabbitmq_process_open_fds * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "File descriptors available", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "When this value reaches zero, new connections will not be accepted.\n\nClient libraries, peer nodes and CLI tools will not be able to connect when the node runs out of available file descriptors.\n\n* [Networking and RabbitMQ](https://www.rabbitmq.com/networking.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 500 + }, + { + "color": "transparent", + "value": 1000 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 15 + }, + "id": 5, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "(rabbitmq_process_max_tcp_sockets * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) -\n(rabbitmq_process_open_tcp_sockets * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "TCP sockets available", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 27, + "panels": [], + "title": "QUEUED MESSAGES", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total number of ready messages ready to be delivered to consumers.\n\nAim to keep this value as low as possible. RabbitMQ behaves best when messages are flowing through it. It's OK for publishers to occasionally outpace consumers, but the expectation is that consumers will eventually process all ready messages.\n\nIf this metric keeps increasing, your system will eventually run out of memory and/or disk space. Consider using TTL or Queue Length Limit to prevent unbounded message growth.\n\n* [Queues](https://www.rabbitmq.com/queues.html)\n* [Consumers](https://www.rabbitmq.com/consumers.html)\n* [Queue Length Limit](https://www.rabbitmq.com/maxlength.html)\n* [Time-To-Live and Expiration](https://www.rabbitmq.com/ttl.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 9, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rabbitmq_queue_messages_ready * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Messages ready to be delivered to consumers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The total number of messages that are either in-flight to consumers, currently being processed by consumers or simply waiting for the consumer acknowledgements to be processed by the queue. Until the queue processes the message acknowledgement, the message will remain unacknowledged.\n\n* [Queues](https://www.rabbitmq.com/queues.html)\n* [Confirms and Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [Consumer Prefetch](https://www.rabbitmq.com/consumer-prefetch.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 19, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rabbitmq_queue_messages_unacked * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Messages pending consumer acknowledgement", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 11, + "panels": [], + "title": "INCOMING MESSAGES", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The incoming message rate before any routing rules are applied.\n\nIf this value is lower than the number of messages published to queues, it may indicate that some messages are delivered to more than one queue.\n\nIf this value is higher than the number of messages published to queues, messages cannot be routed and will either be dropped or returned to publishers.\n\n* [Publishers](https://www.rabbitmq.com/publishers.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 13, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_received_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Messages published / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of messages confirmed by the broker to publishers. Publishers must opt-in to receive message confirmations.\n\nIf this metric is consistently at zero it may suggest that publisher confirms are not used by clients. The safety of published messages is likely to be at risk.\n\n* [Publisher Confirms](https://www.rabbitmq.com/confirms.html#publisher-confirms)\n* [Publisher Confirms and Data Safety](https://www.rabbitmq.com/publishers.html#data-safety)\n* [When Will Published Messages Be Confirmed by the Broker?](https://www.rabbitmq.com/confirms.html#when-publishes-are-confirmed)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 18, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_confirmed_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Messages confirmed to publishers / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of messages received from publishers and successfully routed to the master queue replicas.\n\n* [Queues](https://www.rabbitmq.com/queues.html)\n* [Publishers](https://www.rabbitmq.com/publishers.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 61, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_routed_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Messages routed to queues / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of messages received from publishers that have publisher confirms enabled and the broker has not confirmed yet.\n\n* [Publishers](https://www.rabbitmq.com/publishers.html)\n* [Confirms and Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [When Will Published Messages Be Confirmed by the Broker?](https://www.rabbitmq.com/confirms.html#when-publishes-are-confirmed)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 12, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_received_confirm_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"} - \nrate(rabbitmq_global_messages_confirmed_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}\n) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Messages unconfirmed to publishers / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of messages that cannot be routed and are dropped. \n\nAny value above zero means message loss and likely suggests a routing problem on the publisher end.\n\n* [Unroutable Message Handling](https://www.rabbitmq.com/publishers.html#unroutable)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/rabbit/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 34, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_unroutable_dropped_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Unroutable messages dropped / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of messages that cannot be routed and are returned back to publishers.\n\nSustained values above zero may indicate a routing problem on the publisher end.\n\n* [Unroutable Message Handling](https://www.rabbitmq.com/publishers.html#unroutable)\n* [When Will Published Messages Be Confirmed by the Broker?](https://www.rabbitmq.com/confirms.html#when-publishes-are-confirmed)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/rabbit/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 16, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_unroutable_returned_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Unroutable messages returned to publishers / s", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 29, + "panels": [], + "title": "OUTGOING MESSAGES", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of messages delivered to consumers. It includes messages that have been redelivered.\n\nThis metric does not include messages that have been fetched by consumers using `basic.get` (consumed by polling).\n\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 14, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(\n (rate(rabbitmq_global_messages_delivered_consume_auto_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) +\n (rate(rabbitmq_global_messages_delivered_consume_manual_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})\n) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Messages delivered / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of messages that have been redelivered to consumers. It includes messages that have been requeued automatically and redelivered due to channel exceptions or connection closures.\n\nHaving some redeliveries is expected, but if this metric is consistently non-zero, it is worth investigating why.\n\n* [Negative Acknowledgement and Requeuing of Deliveries](https://www.rabbitmq.com/confirms.html#consumer-nacks-requeue)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "orange", + "value": 20 + }, + { + "color": "red", + "value": 100 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 15, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_redelivered_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Messages redelivered / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of message deliveries to consumers that use manual acknowledgement mode.\n\nWhen this mode is used, RabbitMQ waits for consumers to acknowledge messages before more messages can be delivered.\n\nThis is the safest way of consuming messages.\n\n* [Consumer Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [Consumer Prefetch](https://www.rabbitmq.com/consumer-prefetch.html)\n* [Consumer Acknowledgement Modes, Prefetch and Throughput](https://www.rabbitmq.com/confirms.html#channel-qos-prefetch-throughput)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 20, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_delivered_consume_manual_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Messages delivered with manual ack / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of message deliveries to consumers that use automatic acknowledgement mode.\n\nWhen this mode is used, RabbitMQ does not wait for consumers to acknowledge message deliveries.\n\nThis mode is fire-and-forget and does not offer any delivery safety guarantees. It tends to provide higher throughput and it may lead to consumer overload and higher consumer memory usage.\n\n* [Consumer Acknowledgement Modes, Prefetch and Throughput](https://www.rabbitmq.com/confirms.html#channel-qos-prefetch-throughput)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 21, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_delivered_consume_auto_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Messages delivered auto ack / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of message acknowledgements coming from consumers that use manual acknowledgement mode.\n\n* [Consumer Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [Consumer Prefetch](https://www.rabbitmq.com/consumer-prefetch.html)\n* [Consumer Acknowledgement Modes, Prefetch and Throughput](https://www.rabbitmq.com/confirms.html#channel-qos-prefetch-throughput)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 52 + }, + "id": 22, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_acknowledged_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Messages acknowledged / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of messages delivered to polling consumers that use automatic acknowledgement mode.\n\nThe use of polling consumers is highly inefficient and therefore strongly discouraged.\n\n* [Fetching individual messages](https://www.rabbitmq.com/consumers.html#fetching)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/rabbit/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 52 + }, + "id": 24, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_delivered_get_auto_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Polling operations with auto ack / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of polling consumer operations that yield no result.\n\nAny value above zero means that RabbitMQ resources are wasted by polling consumers.\n\nCompare this metric to the other polling consumer metrics to see the inefficiency rate.\n\nThe use of polling consumers is highly inefficient and therefore strongly discouraged.\n\n* [Fetching individual messages](https://www.rabbitmq.com/consumers.html#fetching)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/rabbit/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 57 + }, + "id": 25, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_get_empty_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Polling operations that yield no result / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of messages delivered to polling consumers that use manual acknowledgement mode.\n\nThe use of polling consumers is highly inefficient and therefore strongly discouraged.\n\n* [Fetching individual messages](https://www.rabbitmq.com/consumers.html#fetching)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/rabbit/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 57 + }, + "id": 23, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_global_messages_delivered_get_manual_ack_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Polling operations with manual ack / s", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 62 + }, + "id": 53, + "panels": [], + "title": "QUEUES", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total number of queue masters per node. \n\nThis metric makes it easy to see sub-optimal queue distribution in a cluster.\n\n* [Queue Masters, Data Locality](https://www.rabbitmq.com/ha.html#master-migration-data-locality)\n* [Queues](https://www.rabbitmq.com/queues.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 63 + }, + "id": 57, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "rabbitmq_queues * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Total queues", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of queue declarations performed by clients.\n\nLow sustained values above zero are to be expected. High rates may be indicative of queue churn or high rates of connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [Queues](https://www.rabbitmq.com/queues.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "orange", + "value": 2 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 63 + }, + "id": 58, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_queues_declared_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Queues declared / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of new queues created (as opposed to redeclarations).\n\nLow sustained values above zero are to be expected. High rates may be indicative of queue churn or high rates of connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [Queues](https://www.rabbitmq.com/queues.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "orange", + "value": 2 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 63 + }, + "id": 60, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_queues_created_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Queues created / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of queues deleted.\n\nLow sustained values above zero are to be expected. High rates may be indicative of queue churn or high rates of connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [Queues](https://www.rabbitmq.com/queues.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "orange", + "value": 2 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 63 + }, + "id": 59, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_queues_deleted_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Queues deleted / s", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 68 + }, + "id": 51, + "panels": [], + "title": "CHANNELS", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total number of channels on all currently opened connections.\n\nIf this metric grows monotonically it is highly likely a channel leak in one of the applications. Confirm channel leaks by using the _Channels opened_ and _Channels closed_ metrics.\n\n* [Channel Leak](https://www.rabbitmq.com/channels.html#channel-leaks)\n* [Channels](https://www.rabbitmq.com/channels.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 69 + }, + "id": 54, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "rabbitmq_channels * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Total channels", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of new channels opened by applications across all connections. Channels are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of channel churn or mass connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [High Channel Churn](https://www.rabbitmq.com/channels.html#high-channel-churn)\n* [Channels](https://www.rabbitmq.com/channels.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "orange", + "value": 2 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 69 + }, + "id": 55, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_channels_opened_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Channels opened / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of channels closed by applications across all connections. Channels are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of channel churn or mass connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [High Channel Churn](https://www.rabbitmq.com/channels.html#high-channel-churn)\n* [Channels](https://www.rabbitmq.com/channels.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "orange", + "value": 2 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 69 + }, + "id": 56, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_channels_closed_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Channels closed / s", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 74 + }, + "id": 46, + "panels": [], + "title": "CONNECTIONS", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total number of client connections.\n\nIf this metric grows monotonically it is highly likely a connection leak in one of the applications. Confirm connection leaks by using the _Connections opened_ and _Connections closed_ metrics.\n\n* [Connection Leak](https://www.rabbitmq.com/connections.html#monitoring)\n* [Connections](https://www.rabbitmq.com/connections.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 75 + }, + "id": 47, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "rabbitmq_connections * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Total connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of new connections opened by clients. Connections are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of connection churn or mass connection recovery.\n\n* [Connection Leak](https://www.rabbitmq.com/connections.html#monitoring)\n* [Connections](https://www.rabbitmq.com/connections.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "orange", + "value": 2 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 75 + }, + "id": 48, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_connections_opened_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Connections opened / s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The rate of connections closed. Connections are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of connection churn or mass connection recovery.\n\n* [Connections](https://www.rabbitmq.com/connections.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "orange", + "value": 2 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 75 + }, + "id": 49, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "expr": "sum(rate(rabbitmq_connections_closed_total[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Connections closed / s", + "type": "timeseries" + } + ], + "refresh": "15s", + "schemaVersion": 34, + "style": "dark", + "tags": [ + "rabbitmq-prometheus" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "datasource": "PBFA97CFB590B2093", + "hide": 2, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(rabbitmq_identity_info, namespace)", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(rabbitmq_identity_info, namespace)", + "refId": "Prometheus-namespace-Variable-Query" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(rabbitmq_identity_info{namespace=\"$namespace\"}, rabbitmq_cluster)", + "hide": 0, + "includeAll": false, + "label": "RabbitMQ Cluster", + "multi": false, + "name": "rabbitmq_cluster", + "options": [], + "query": { + "query": "label_values(rabbitmq_identity_info{namespace=\"$namespace\"}, rabbitmq_cluster)", + "refId": "Prometheus-rabbitmq_cluster-Variable-Query" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "15s", + "30s", + "1m", + "5m", + "10m" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "RabbitMQ-Overview", + "uid": "Kn5xm-gZk", + "version": 20220805, + "weekStart": "", + "gnetId": 10991 +} \ No newline at end of file diff --git a/deployments/configs/grafana/provisioning/dashboards/dashboard.yml b/deployments/configs/grafana/provisioning/dashboards/dashboard.yml new file mode 100644 index 00000000..f1fb0bb2 --- /dev/null +++ b/deployments/configs/grafana/provisioning/dashboards/dashboard.yml @@ -0,0 +1,14 @@ +# https://grafana.com/docs/grafana/latest/administration/provisioning/#dashboards +apiVersion: 1 + +providers: + - name: "default" + orgId: 1 + folder: "" + type: file + disableDeletion: false + editable: true + allowUiUpdates: true + updateIntervalSeconds: 5 # how often Grafana will scan for changed dashboards + options: + path: /var/lib/grafana/dashboards # path to dashboards on disk \ No newline at end of file diff --git a/deployments/configs/grafana/provisioning/datasources/datasource.yml b/deployments/configs/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 00000000..5b04359b --- /dev/null +++ b/deployments/configs/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,88 @@ +# https://grafana.com/docs/grafana/latest/administration/provisioning/ +# https://github.com/grafana/tempo/blob/main/example/docker-compose/shared/grafana-datasources.yaml +# https://github.com/grafana/intro-to-mltp/blob/main/grafana/provisioning/datasources/datasources.yaml + +apiVersion: 1 + +datasources: + # https://github.com/grafana/tempo/blob/main/example/docker-compose/shared/grafana-datasources.yaml + - name: Prometheus + type: prometheus + typeName: Prometheus + uid: prometheus-uid + access: proxy + orgId: 1 + url: http://prometheus:9090 + basicAuth: false + isDefault: true + readOnly: false + user: '' + database: '' + version: 1 + editable: false + jsonData: + httpMethod: GET + + - name: Jaeger + type: jaeger + access: proxy + url: http://jaeger-all-in-one:16686 + editable: false + uid: jaeger-uid + + - name: Zipkin + type: zipkin + access: proxy + url: http://zipkin-all-in-one:9411 + editable: false + uid: zipkin-uid + + # https://github.com/grafana/tempo/blob/main/example/docker-compose/shared/grafana-datasources.yaml + - name: Tempo + type: tempo + access: proxy + orgId: 1 + url: http://tempo:3200 + basicAuth: false + isDefault: false + version: 1 + editable: false + apiVersion: 1 + uid: tempo-uid + jsonData: + httpMethod: GET + serviceMap: + datasourceUid: prometheus-uid + streamingEnabled: + search: true + + #https://github.com/grafana/intro-to-mltp/blob/main/grafana/provisioning/datasources/datasources.yaml + - name: Loki + type: loki + access: proxy + uid: loki-uid + url: http://loki:3100 + user: '' + database: '' + readOnly: false + jsonData: + derivedFields: + - datasourceUid: tempo-uid + matcherRegex: "^.*?traceI[d|D]=(\\w+).*$" + name: traceId + url: '$${__value.raw}' + + - name: Kibana + type: elasticsearch + url: http://elasticsearch:9200 + access: proxy + isDefault: false + uid: kibana-uid + jsonData: + esVersion: 7 + timeField: "@timestamp" + maxConcurrentShardRequests: 256 + interval: Daily + logMessageField: "message" # Optional: Field for log messages + logLevelField: "level" # Optional: Field for log levels + editable: true \ No newline at end of file diff --git a/deployments/configs/loki-config.yaml b/deployments/configs/loki-config.yaml new file mode 100644 index 00000000..414d7626 --- /dev/null +++ b/deployments/configs/loki-config.yaml @@ -0,0 +1,44 @@ +# https://grafana.com/docs/loki/latest/configure/examples/configuration-examples/ +# https://github.com/grafana/loki/issues/2018#issuecomment-970789233 +# https://grafana.com/docs/opentelemetry/collector/send-logs-to-loki/ +# https://github.com/grafana/loki/blob/main/examples/getting-started/loki-config.yaml +# https://github.com/grafana/loki/blob/main/cmd/loki/loki-local-config.yaml +# https://grafana.com/docs/loki/latest/configure/examples/configuration-examples/#1-local-configuration-exampleyaml + +--- +# https://grafana.com/docs/loki/latest/configure/examples/configuration-examples/#1-local-configuration-exampleyaml +auth_enabled: false + +# This is a complete configuration to deploy Loki backed by the filesystem. +# The index will be shipped to the storage via tsdb-shipper. +server: + http_listen_port: 3100 + +common: + ring: + instance_addr: 127.0.0.1 + kvstore: + store: inmemory + replication_factor: 1 + path_prefix: /tmp/loki + +schema_config: + configs: + - from: 2020-05-15 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +storage_config: + filesystem: + directory: /tmp/loki/chunks + +# https://grafana.com/docs/loki/latest/send-data/otel/ +# https://grafana.com/docs/loki/latest/send-data/otel/#changing-the-default-mapping-of-otlp-to-loki-format +limits_config: + # this attribute should be `true` when we use `otlphttp/loki`, but if we want to use `loki component` from `opentelemetry-collector-contrib` it should be false. + allow_structured_metadata: true + diff --git a/deployments/configs/otel-collector-config.yaml b/deployments/configs/otel-collector-config.yaml new file mode 100644 index 00000000..ea6ef405 --- /dev/null +++ b/deployments/configs/otel-collector-config.yaml @@ -0,0 +1,131 @@ +# ref: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/examples/demo/otel-collector-config.yaml +# https://opentelemetry.io/docs/collector/configuration/ +# https://opentelemetry.io/docs/collector/architecture/ +# https://betterstack.com/community/guides/observability/opentelemetry-collector/ +# https://signoz.io/blog/opentelemetry-collector-complete-guide/ + +# This configuration sets up an OpenTelemetry Collector that receives trace data via the OTLP protocol over HTTP on port 4318, applies batch processing, and then exports the processed traces +# to exporter components like `Jaeger` endpoint located at `jaeger-all-in-one:4317`. It also includes a health_check extension for monitoring the collector's status. + +# Receivers in the OpenTelemetry Collector are components that collect telemetry data (traces, metrics, and logs) from various sources, such as instrumented applications or agents. +# They act as entry points, converting incoming data into OpenTelemetry's internal format for processing and export. +# https://betterstack.com/community/guides/observability/opentelemetry-collector/#exploring-the-opentelemetry-collector-components +# https://opentelemetry.io/docs/collector/architecture/#receivers +# https://github.com/open-telemetry/opentelemetry-collector/blob/main/receiver/README.md +# https://opentelemetry.io/docs/collector/configuration/#receivers +receivers: + # supported receivers + # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver + # instead of specifying details explicitly we can just use `otlp` and it uses both grpc and http + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 +# prometheus: +# config: +# scrape_configs: +# - job_name: 'node-exporter' +# scrape_interval: 10s +# static_configs: +# - targets: [ 'node-exporter:9100' ] + + +# Processors in the OpenTelemetry Collector modify and enhance telemetry data by filtering, transforming, enriching, or batching it to prepare it for export. +# https://github.com/open-telemetry/opentelemetry-collector/tree/main/processor +processors: + batch: # Batches logs for better performance + +# - Exporters in the OpenTelemetry Collector send processed telemetry data to backend systems like observability platforms, databases, or cloud services for storage, visualization, and analysis. +# - The `key` follows the `type/name` format, where `type` specifies the exporter `type` (e.g., otlp, kafka, prometheus), and `name` (optional) can be appended to provide a unique name for multiple instance of the same type +# https://betterstack.com/community/guides/observability/opentelemetry-collector/#exploring-the-opentelemetry-collector-components +# https://opentelemetry.io/docs/collector/architecture/#exporters +# https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter +# https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter +# https://opentelemetry.io/docs/collector/configuration/#exporters +exporters: + + # valid values: [prometheusremotewrite zipkin otlphttp file kafka prometheus debug nop otlp opencensus] + + # Prometheus exporter metrics + prometheus: + endpoint: "0.0.0.0:8889" + + prometheusremotewrite: + endpoint: "http://prometheus:9090/api/v1/write" + + # https://grafana.com/docs/loki/latest/send-data/otel/ + # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/lokiexporter/README.md + otlphttp/loki: + endpoint: "http://loki:3100/otlp" + tls: + insecure: true + + # # we can also use `loki component` from `opentelemetry-collector-contrib` if we don't want to use builtin `otlphttp` exporter type and `http://loki:3100/otlp` loki endpoint + # loki: + # endpoint: "http://loki:3100/loki/api/v1/push" + # tls: + # insecure: true + + debug: + + # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/elasticsearchexporter + # using `elasticsearch` from `opentelemetry-collector-contrib` components because it doesn't exist in `opentelemetry-collector` + elasticsearch: + endpoint: "http://elasticsearch:9200" + + zipkin: + endpoint: "http://zipkin-all-in-one:9411/api/v2/spans" + format: proto + + # export collected telemetry traces to jaeger OTLP grpc port, we can send data to other available endpoints and ports on jaeger as well + otlp/jaeger: + endpoint: "http://jaeger-all-in-one:4317" + tls: + insecure: true + + otlp/tempo: + endpoint: "http://tempo:4317" + tls: + insecure: true + + # seq-otlp: + # endpoint: "http://seq:5341/ingest/otlp" + +# https://opentelemetry.io/docs/collector/configuration/#extensions +# https://github.com/open-telemetry/opentelemetry-collector/blob/main/extension/README.md +extensions: + pprof: + endpoint: 0.0.0.0:1888 + zpages: + endpoint: 0.0.0.0:55679 + health_check: + endpoint: 0.0.0.0:13133 + +# - The service section is used to configure what components are enabled in the Collector based on the configuration found in the receivers, processors, exporters, and extensions sections. +# - If a component is configured, but not defined within the service section, then it’s not enabled. +# https://betterstack.com/community/guides/observability/opentelemetry-collector/#exploring-the-opentelemetry-collector-components +# https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/README.md +# https://opentelemetry.io/docs/collector/architecture/ +# https://opentelemetry.io/docs/collector/configuration/#service +service: + # The `service.extensions` subsection determines which of the configured extensions will be enabled + extensions: [pprof, zpages, health_check] + # The `service.pipeline` Each pipeline starts with one or more receivers collecting data, which is then processed sequentially by processors (applying transformations, filtering, or sampling). + # The processed data is finally sent to all configured exporters, ensuring each receives a copy. Components must be pre-configured in their respective sections before being used in a pipeline. + # pipeline activate predefined components, defined components are disabled by default + pipelines: + traces: + receivers: [otlp] + processors: [batch] + # https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/#exporter-selection + exporters: [debug, zipkin, otlp/jaeger, otlp/tempo] + metrics: + receivers: [otlp] + processors: [batch] + exporters: [debug, prometheusremotewrite, prometheus] + logs: + receivers: [otlp] + processors: [batch] + exporters: [otlphttp/loki, elasticsearch] \ No newline at end of file diff --git a/deployments/configs/prometheus.yaml b/deployments/configs/prometheus.yaml new file mode 100644 index 00000000..1928aafd --- /dev/null +++ b/deployments/configs/prometheus.yaml @@ -0,0 +1,48 @@ +# ref: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/examples/demo/prometheus.yaml +# https://prometheus.io/docs/introduction/first_steps/ +# https://grafana.com/docs/grafana-cloud/send-data/metrics/metrics-prometheus/prometheus-config-examples/docker-compose-linux/ + +global: + scrape_interval: 5s + +scrape_configs: + # when we use otel-collector we should comment other jobs in prometheus config, and we read configs from `otel-collector-config` + - job_name: "otel-collector" + scrape_interval: 10s + static_configs: + # otel-collector Prometheus exporter metrics + - targets: [ 'otel-collector:8889' ] + - targets: [ 'otel-collector:8888' ] + + - job_name: "prometheus" + static_configs: + - targets: ["prometheus:9090"] + +# # https://prometheus.io/docs/guides/node-exporter/ +# # https://grafana.com/docs/grafana-cloud/send-data/metrics/metrics-prometheus/prometheus-config-examples/docker-compose-linux/ +# - job_name: "node-exporter" +# static_configs: +# - targets: [ 'node-exporter:9100' ] + +# # if we don't use otel collector we should uncomment this +# # scrap application metrics +# # http://localhost:4000/metrics by AddPrometheusExporter() +# - job_name: vertical-slice-template-api +# scrape_interval: 10s +# metrics_path: /metrics +# static_configs: +# - targets: ['host.docker.internal:4000'] +# +# # if we don't use otel collector we should uncomment this +# # scrap application health metrics +# # http://localhost:4000/health/metrics by AddPrometheusExporter() +# - job_name: vertical-slice-template-api-healthchecks +# scrape_interval: 10s +# metrics_path: /health/metrics +# static_configs: +# - targets: ['host.docker.internal:4000'] + +## https://github.com/grafana/tempo/blob/main/example/docker-compose/shared/prometheus.yaml +# - job_name: 'tempo' +# static_configs: +# - targets: [ 'tempo:3200' ] \ No newline at end of file diff --git a/deployments/configs/tempo.yaml b/deployments/configs/tempo.yaml new file mode 100644 index 00000000..fcb2ed38 --- /dev/null +++ b/deployments/configs/tempo.yaml @@ -0,0 +1,49 @@ +# https://grafana.com/docs/tempo/latest/configuration/ +# https://github.com/grafana/tempo/blob/main/example/docker-compose/local/tempo.yaml + +# https://github.com/grafana/tempo/blob/main/example/docker-compose/shared/tempo.yaml +stream_over_http_enabled: true +server: + http_listen_port: 3200 + log_level: info + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: "tempo:4317" + +ingester: + max_block_duration: 5m # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally + +compactor: + compaction: + block_retention: 1h # overall Tempo trace retention. set for demo purposes + +metrics_generator: + registry: + external_labels: + source: tempo + cluster: docker-compose + storage: + path: /var/tempo/generator/wal + remote_write: + - url: http://prometheus:9090/api/v1/write + send_exemplars: true + traces_storage: + path: /var/tempo/generator/traces + +storage: + trace: + backend: local # backend configuration to use + wal: + path: /var/tempo/wal # where to store the wal locally + local: + path: /var/tempo/blocks + +overrides: + defaults: + metrics_generator: + processors: [service-graphs, span-metrics, local-blocks] # enables metrics generator + generate_native_histograms: both \ No newline at end of file diff --git a/deployments/docker-compose/docker-compose.infrastructure.yaml b/deployments/docker-compose/docker-compose.infrastructure.yaml new file mode 100644 index 00000000..630517ea --- /dev/null +++ b/deployments/docker-compose/docker-compose.infrastructure.yaml @@ -0,0 +1,362 @@ +# ref: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/examples/demo/docker-compose.yaml +# ref: https://github.com/joaofbantunes/DotNetMicroservicesObservabilitySample/blob/main/docker-compose.yml +# ref: https://github.com/oskardudycz/EventSourcing.NetCore/blob/main/docker-compose.yml +# https://github.com/grafana/intro-to-mltp +# https://stackoverflow.com/questions/65272764/ports-are-not-available-listen-tcp-0-0-0-0-50070-bind-an-attempt-was-made-to + + +name: booking-microservices + +services: + ####################################################### + # rabbitmq + ####################################################### + rabbitmq: + image: rabbitmq:management + container_name: rabbitmq + restart: unless-stopped + ports: + - "5672:5672" + - "15672:15672" + # volumes: + # - rabbitmq:/var/lib/rabbitmq + networks: + - infrastructure + + ####################################################### + # postgres + ####################################################### + postgres: + image: postgres:latest + container_name: postgres + restart: unless-stopped + ports: + - '5432:5432' + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + command: + - "postgres" + - "-c" + - "wal_level=logical" + - "-c" + - "max_prepared_transactions=10" + volumes: + - postgres-data:/var/lib/postgresql/data + networks: + - infrastructure + + + ####################################################### + # EventStoreDB + ####################################################### + eventstore: + container_name: eventstore + image: eventstore/eventstore:latest + restart: unless-stopped + environment: + - EVENTSTORE_CLUSTER_SIZE=1 + - EVENTSTORE_RUN_PROJECTIONS=All + - EVENTSTORE_START_STANDARD_PROJECTIONS=True + - EVENTSTORE_HTTP_PORT=2113 + - EVENTSTORE_INSECURE=True + - EVENTSTORE_ENABLE_ATOM_PUB_OVER_HTTP=True + ports: + - "2113:2113" + networks: + - infrastructure + + + ####################################################### + # Mongo + ####################################################### + mongo: + image: mongo:latest + container_name: mongo + restart: unless-stopped + # environment: + # - MONGO_INITDB_ROOT_USERNAME=root + # - MONGO_INITDB_ROOT_PASSWORD=secret + ports: + - 27017:27017 + networks: + - infrastructure + + + ####################################################### + # Redis + ####################################################### + redis: + image: redis + container_name: redis + restart: unless-stopped + ports: + - 6379:6379 + networks: + - infrastructure + + + ####################################################### + # jaeger + # https://www.jaegertracing.io/docs/1.64/deployment/ + # https://www.jaegertracing.io/docs/1.6/deployment/ + ####################################################### + jaeger-all-in-one: + image: jaegertracing/all-in-one:latest + container_name: jaeger-all-in-one + restart: unless-stopped + ports: + - "6831:6831/udp" # UDP port for Jaeger agent + - "16686:16686" # endpoints and Jaeger UI + - "14268:14268" # HTTP port for accept trace spans directly from clients + - "14317:4317" # OTLP gRPC receiver for jaeger + - "14318:4318" # OTLP http receiver for jaeger + # - "9411" # Accepts Zipkin spans - /api/v2/spans + networks: + - infrastructure + + + ####################################################### + # zipkin + # https://zipkin.io/pages/quickstart + ####################################################### + zipkin-all-in-one: + image: openzipkin/zipkin:latest + container_name: zipkin-all-in-one + restart: unless-stopped + ports: + - "9411:9411" + networks: + - infrastructure + + + ####################################################### + # otel-collector + # https://opentelemetry.io/docs/collector/installation/ + # https://github.com/open-telemetry/opentelemetry-collector + # https://github.com/open-telemetry/opentelemetry-collector-contrib + # we can use none contrib docker `otel/opentelemetry-collector` version from `https://github.com/open-telemetry/opentelemetry-collector` repository but, + # if we need more components like `elasticsearch` we should use `otel/opentelemetry-collector-contrib` image of `https://github.com/open-telemetry/opentelemetry-collector-contrib` repository. + ####################################################### + otel-collector: + image: otel/opentelemetry-collector-contrib:latest + container_name: otel-collector + restart: unless-stopped + command: ["--config=/etc/otelcol-contrib/config.yaml"] + volumes: + - ./../configs/otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml + ports: + - "11888:1888" # pprof extension + - "8888:8888" # Prometheus metrics exposed by the Collector + - "8889:8889" # Prometheus exporter metrics + - "13133:13133" # health_check extension + - "4317:4317" # OTLP gRPC receiver + - "4318:4318" # OTLP http receiver + - "55679:55679" # zpages extension + networks: + - infrastructure + + + ####################################################### + # prometheus + # https://prometheus.io/docs/introduction/first_steps/ + # https://prometheus.io/docs/prometheus/3.1/installation/ + # https://grafana.com/docs/grafana-cloud/send-data/metrics/metrics-prometheus/prometheus-config-examples/docker-compose-linux/ + ####################################################### + prometheus: + image: prom/prometheus:latest + restart: unless-stopped + ports: + - "9090:9090" + volumes: + - ./../configs/prometheus.yaml:/etc/prometheus/prometheus.yml + # to passe one flag, such as "--log.level=debug" or "--web.enable-remote-write-receiver", we need to override the whole command, as we can't just pass one extra argument + command: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--web.console.libraries=/usr/share/prometheus/console_libraries" + - "--web.console.templates=/usr/share/prometheus/consoles" + # need this for the OpenTelemetry collector to be able to put metrics into Prometheus + - "--web.enable-remote-write-receiver" + # - "--log.level=debug" + networks: + - infrastructure + + + ####################################################### + # node-exporter + # https://prometheus.io/docs/guides/node-exporter/ + # https://grafana.com/docs/grafana-cloud/send-data/metrics/metrics-prometheus/prometheus-config-examples/docker-compose-linux/ + ####################################################### + node-exporter: + image: prom/node-exporter:latest + container_name: node-exporter + restart: unless-stopped + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + command: + - '--path.procfs=/host/proc' + - '--path.rootfs=/rootfs' + - '--path.sysfs=/host/sys' + ports: + - "9101:9100" + networks: + - infrastructure + + + ####################################################### + # grafana + # https://grafana.com/docs/grafana/latest/administration/provisioning/ + # https://grafana.com/docs/grafana/latest/setup-grafana/installation/docker/ + # https://grafana.com/docs/grafana/latest/setup-grafana/configure-docker/ + # https://github.com/grafana/intro-to-mltp/blob/main/grafana/provisioning/datasources/datasources.yaml + ####################################################### + grafana: + image: grafana/grafana:latest + container_name: grafana + restart: unless-stopped + environment: + - GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-simple-json-datasource + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor + # - GF_AUTH_ANONYMOUS_ENABLED=true + # - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + # - GF_AUTH_DISABLE_LOGIN_FORM=true + depends_on: + - prometheus + ports: + - "3000:3000" + volumes: + - ./../configs/grafana/provisioning:/etc/grafana/provisioning + - ./../configs/grafana/dashboards:/var/lib/grafana/dashboards + ## https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/ + # - ./../configs/grafana/grafana.ini:/etc/grafana/grafana.ini + networks: + - infrastructure + + + ####################################################### + # tempo + # https://github.com/grafana/tempo/blob/main/example/docker-compose/otel-collector/docker-compose.yaml + # https://github.com/grafana/tempo/blob/main/example/docker-compose/shared + # https://github.com/grafana/tempo/blob/main/example/docker-compose/local + # https://github.com/grafana/tempo/tree/main/example/docker-compose + ####################################################### + tempo: + image: grafana/tempo:latest + container_name: tempo + restart: unless-stopped + command: [ "-config.file=/etc/tempo.yaml" ] + volumes: + - ./../configs/tempo.yaml:/etc/tempo.yaml + ports: + - "3200" # tempo + - "24317:4317" # otlp grpc + - "24318:4318" # otlp http + networks: + - infrastructure + + + ####################################################### + # loki + # https://grafana.com/docs/opentelemetry/collector/send-logs-to-loki/ + # https://github.com/grafana/loki/blob/main/production/docker-compose.yaml + # https://github.com/grafana/loki/blob/main/examples/getting-started/docker-compose.yaml + ####################################################### + loki: + image: grafana/loki:latest + hostname: loki + container_name: loki + ports: + - "3100:3100" + command: -config.file=/etc/loki/local-config.yaml + volumes: + - ./../configs/loki-config.yaml:/etc/loki/local-config.yaml + networks: + - infrastructure + + + ####################################################### + # elasticsearch + # https://www.elastic.co/guide/en/elasticsearch/reference/7.17/docker.html#docker-compose-file + ####################################################### + elasticsearch: + container_name: elasticsearch + restart: unless-stopped + image: docker.elastic.co/elasticsearch/elasticsearch:8.17.0 + environment: + - discovery.type=single-node + - cluster.name=docker-cluster + - node.name=docker-node + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - xpack.security.enabled=false + - xpack.security.http.ssl.enabled=false + - xpack.security.transport.ssl.enabled=false + - network.host=0.0.0.0 + - http.port=9200 + - transport.host=localhost + - bootstrap.memory_lock=true + - cluster.routing.allocation.disk.threshold_enabled=false + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - elastic-data:/usr/share/elasticsearch/data + ports: + - ${ELASTIC_HOST_PORT:-9200}:${ELASTIC_PORT:-9200} + - 9300:9300 + networks: + - infrastructure + + + ####################################################### + # kibana + # https://www.elastic.co/guide/en/kibana/current/docker.html + ####################################################### + kibana: + image: docker.elastic.co/kibana/kibana:8.17.0 + container_name: kibana + restart: unless-stopped + environment: + - ELASTICSEARCH_HOSTS=http://elasticsearch:9200 + ports: + - ${KIBANA_HOST_PORT:-5601}:${KIBANA_PORT:-5601} + depends_on: + - elasticsearch + networks: + - infrastructure + + + ####################################################### + # cadvisor + ####################################################### + cadvisor: + image: gcr.io/cadvisor/cadvisor:latest + container_name: cadvisor + restart: unless-stopped + ports: + - "8080:8080" + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + - /dev/disk/:/dev/disk:ro + devices: + - /dev/kmsg + networks: + - infrastructure + + +networks: + infrastructure: + name: infrastructure + driver: bridge + +volumes: + elastic-data: + postgres-data: \ No newline at end of file diff --git a/deployments/docker-compose/docker-compose.yaml b/deployments/docker-compose/docker-compose.yaml index e6ee7f73..4c288dde 100644 --- a/deployments/docker-compose/docker-compose.yaml +++ b/deployments/docker-compose/docker-compose.yaml @@ -1,79 +1,49 @@ -version: "3.3" -services: - - ####################################################### - # Postgres - ###################################################### - postgres: - image: postgres:latest - container_name: postgres - restart: unless-stopped - ports: - - '5432:5432' - environment: - - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=postgres - command: - - "postgres" - - "-c" - - "wal_level=logical" - - "-c" - - "max_prepared_transactions=10" - networks: - - booking - - - - ####################################################### - # SqlServer - ####################################################### - # sql: - # container_name: sql - # image: mcr.microsoft.com/mssql/server - # restart: unless-stopped - # ports: - # - "1433:1433" - # environment: - # SA_PASSWORD: "Password@1234" - # ACCEPT_EULA: "Y" +name: booking-microservices +services: - ####################################################### - # Rabbitmq - ####################################################### + ####################################################### + # rabbitmq + ####################################################### rabbitmq: - container_name: rabbitmq image: rabbitmq:management + container_name: rabbitmq restart: unless-stopped ports: - - 5672:5672 - - 15672:15672 + - "5672:5672" + - "15672:15672" + # volumes: + # - rabbitmq:/var/lib/rabbitmq networks: - booking - - ####################################################### - # Jaeger - ####################################################### - jaeger: - image: jaegertracing/all-in-one - container_name: jaeger + ####################################################### + # postgres + ####################################################### + postgres: + image: postgres:latest + container_name: postgres restart: unless-stopped + ports: + - '5432:5432' + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + command: + - "postgres" + - "-c" + - "wal_level=logical" + - "-c" + - "max_prepared_transactions=10" + volumes: + - postgres-data:/var/lib/postgresql/data networks: - booking - ports: - - 5775:5775/udp - - 5778:5778 - - 6831:6831/udp - - 6832:6832/udp - - 9411:9411 - - 14268:14268 - - 16686:16686 - ####################################################### - # EventStoreDB - ####################################################### + ####################################################### + # EventStoreDB + ####################################################### eventstore: container_name: eventstore image: eventstore/eventstore:latest @@ -88,12 +58,12 @@ services: ports: - "2113:2113" networks: - - booking + - booking - ####################################################### - # Mongo - ####################################################### + ####################################################### + # Mongo + ####################################################### mongo: image: mongo:latest container_name: mongo @@ -101,93 +71,260 @@ services: # environment: # - MONGO_INITDB_ROOT_USERNAME=root # - MONGO_INITDB_ROOT_PASSWORD=secret + ports: + - 27017:27017 networks: - booking + + + ####################################################### + # Redis + ####################################################### + redis: + image: redis + container_name: redis + restart: unless-stopped ports: - - 27017:27017 + - 6379:6379 + networks: + - booking - ####################################################### - # Elastic Search - ####################################################### - elasticsearch: - container_name: elasticsearch - image: elasticsearch:7.17.9 + + ####################################################### + # jaeger + # https://www.jaegertracing.io/docs/1.64/deployment/ + # https://www.jaegertracing.io/docs/1.6/deployment/ + ####################################################### + jaeger-all-in-one: + image: jaegertracing/all-in-one:latest + container_name: jaeger-all-in-one restart: unless-stopped ports: - - 9200:9200 - volumes: - - elasticsearch-data:/usr/share/elasticsearch/data - environment: - - xpack.monitoring.enabled=true - - xpack.watcher.enabled=false - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - - discovery.type=single-node + - "6831:6831/udp" # UDP port for Jaeger agent + - "16686:16686" # endpoints and Jaeger UI + - "14268:14268" # HTTP port for accept trace spans directly from clients + - "14317:4317" # OTLP gRPC receiver for jaeger + - "14318:4318" # OTLP http receiver for jaeger + # - "9411" # Accepts Zipkin spans - /api/v2/spans networks: - booking - ####################################################### - # Kibana - ####################################################### - kibana: - container_name: kibana - image: kibana:7.17.9 + + ####################################################### + # zipkin + # https://zipkin.io/pages/quickstart + ####################################################### + zipkin-all-in-one: + image: openzipkin/zipkin:latest + container_name: zipkin-all-in-one restart: unless-stopped ports: - - 5601:5601 - depends_on: - - elasticsearch - environment: - - ELASTICSEARCH_URL=elasticsearch:9200 + - "9411:9411" networks: - booking - ####################################################### - # prometheus - ####################################################### + + ####################################################### + # otel-collector + # https://opentelemetry.io/docs/collector/installation/ + # https://github.com/open-telemetry/opentelemetry-collector + # https://github.com/open-telemetry/opentelemetry-collector-contrib + # we can use none contrib docker `otel/opentelemetry-collector` version from `https://github.com/open-telemetry/opentelemetry-collector` repository but, + # if we need more components like `elasticsearch` we should use `otel/opentelemetry-collector-contrib` image of `https://github.com/open-telemetry/opentelemetry-collector-contrib` repository. + ####################################################### + otel-collector: + image: otel/opentelemetry-collector-contrib:latest + container_name: otel-collector + restart: unless-stopped + command: ["--config=/etc/otelcol-contrib/config.yaml"] + volumes: + - ./../configs/otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml + ports: + - "11888:1888" # pprof extension + - "8888:8888" # Prometheus metrics exposed by the Collector + - "8889:8889" # Prometheus exporter metrics + - "13133:13133" # health_check extension + - "4317:4317" # OTLP gRPC receiver + - "4318:4318" # OTLP http receiver + - "55679:55679" # zpages extension + networks: + - booking + + + ####################################################### + # prometheus + # https://prometheus.io/docs/introduction/first_steps/ + # https://prometheus.io/docs/prometheus/3.1/installation/ + # https://grafana.com/docs/grafana-cloud/send-data/metrics/metrics-prometheus/prometheus-config-examples/docker-compose-linux/ + ####################################################### prometheus: image: prom/prometheus:latest - container_name: prometheus restart: unless-stopped ports: - "9090:9090" - environment: - - TZ=UTC volumes: - - ./monitoring/prom/prometheus.yml:/etc/prometheus/prometheus.yml + - ./../configs/prometheus.yaml:/etc/prometheus/prometheus.yml + # to passe one flag, such as "--log.level=debug" or "--web.enable-remote-write-receiver", we need to override the whole command, as we can't just pass one extra argument + command: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--web.console.libraries=/usr/share/prometheus/console_libraries" + - "--web.console.templates=/usr/share/prometheus/consoles" + # need this for the OpenTelemetry collector to be able to put metrics into Prometheus + - "--web.enable-remote-write-receiver" + # - "--log.level=debug" networks: - booking - ####################################################### - # grafana - ####################################################### + + ####################################################### + # node-exporter + # https://prometheus.io/docs/guides/node-exporter/ + # https://grafana.com/docs/grafana-cloud/send-data/metrics/metrics-prometheus/prometheus-config-examples/docker-compose-linux/ + ####################################################### + node-exporter: + image: prom/node-exporter:latest + container_name: node-exporter + restart: unless-stopped + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + command: + - '--path.procfs=/host/proc' + - '--path.rootfs=/rootfs' + - '--path.sysfs=/host/sys' + ports: + - "9101:9100" + networks: + - booking + + + ####################################################### + # grafana + # https://grafana.com/docs/grafana/latest/administration/provisioning/ + # https://grafana.com/docs/grafana/latest/setup-grafana/installation/docker/ + # https://grafana.com/docs/grafana/latest/setup-grafana/configure-docker/ + # https://github.com/grafana/intro-to-mltp/blob/main/grafana/provisioning/datasources/datasources.yaml + ####################################################### grafana: - image: grafana/grafana + image: grafana/grafana:latest container_name: grafana restart: unless-stopped + environment: + - GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-simple-json-datasource + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor + # - GF_AUTH_ANONYMOUS_ENABLED=true + # - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + # - GF_AUTH_DISABLE_LOGIN_FORM=true + depends_on: + - prometheus ports: - "3000:3000" volumes: - - ./monitoring/grafana-data/data:/var/lib/grafana + - ./../configs/grafana/provisioning:/etc/grafana/provisioning + - ./../configs/grafana/dashboards:/var/lib/grafana/dashboards + ## https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/ + # - ./../configs/grafana/grafana.ini:/etc/grafana/grafana.ini networks: - booking - ####################################################### - # node_exporter - ####################################################### - node_exporter: - image: quay.io/prometheus/node-exporter:latest - container_name: node_exporter + + ####################################################### + # tempo + # https://github.com/grafana/tempo/blob/main/example/docker-compose/otel-collector/docker-compose.yaml + # https://github.com/grafana/tempo/blob/main/example/docker-compose/shared + # https://github.com/grafana/tempo/blob/main/example/docker-compose/local + # https://github.com/grafana/tempo/tree/main/example/docker-compose + ####################################################### + tempo: + image: grafana/tempo:latest + container_name: tempo restart: unless-stopped - command: - - '--path.rootfs=/host' - pid: host + command: [ "-config.file=/etc/tempo.yaml" ] volumes: - - /proc:/host/proc:ro - - /sys:/host/sys:ro - - /:/rootfs:ro + - ./../configs/tempo.yaml:/etc/tempo.yaml + ports: + - "3200" # tempo + - "24317:4317" # otlp grpc + - "24318:4318" # otlp http + networks: + - booking + + + ####################################################### + # loki + # https://grafana.com/docs/opentelemetry/collector/send-logs-to-loki/ + # https://github.com/grafana/loki/blob/main/production/docker-compose.yaml + # https://github.com/grafana/loki/blob/main/examples/getting-started/docker-compose.yaml + ####################################################### + loki: + image: grafana/loki:latest + hostname: loki + container_name: loki + ports: + - "3100:3100" + command: -config.file=/etc/loki/local-config.yaml + volumes: + - ./../configs/loki-config.yaml:/etc/loki/local-config.yaml networks: - booking + + + ####################################################### + # elasticsearch + # https://www.elastic.co/guide/en/elasticsearch/reference/7.17/docker.html#docker-compose-file + ####################################################### + elasticsearch: + container_name: elasticsearch + restart: unless-stopped + image: docker.elastic.co/elasticsearch/elasticsearch:8.17.0 + environment: + - discovery.type=single-node + - cluster.name=docker-cluster + - node.name=docker-node + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - xpack.security.enabled=false + - xpack.security.http.ssl.enabled=false + - xpack.security.transport.ssl.enabled=false + - network.host=0.0.0.0 + - http.port=9200 + - transport.host=localhost + - bootstrap.memory_lock=true + - cluster.routing.allocation.disk.threshold_enabled=false + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - elastic-data:/usr/share/elasticsearch/data + ports: + - ${ELASTIC_HOST_PORT:-9200}:${ELASTIC_PORT:-9200} + - 9300:9300 + networks: + - booking + + ####################################################### + # kibana + # https://www.elastic.co/guide/en/kibana/current/docker.html + ####################################################### + kibana: + image: docker.elastic.co/kibana/kibana:8.17.0 + container_name: kibana + restart: unless-stopped + environment: + - ELASTICSEARCH_HOSTS=http://elasticsearch:9200 + ports: + - ${KIBANA_HOST_PORT:-5601}:${KIBANA_PORT:-5601} + depends_on: + - elasticsearch + networks: + - booking + + ####################################################### # cadvisor ####################################################### @@ -208,6 +345,7 @@ services: networks: - booking + ###################################################### # Gateway ###################################################### @@ -344,9 +482,11 @@ services: networks: booking: - + name: booking + driver: bridge + volumes: - elasticsearch-data: - + elastic-data: + postgres-data: diff --git a/deployments/docker-compose/infrastructure.yaml b/deployments/docker-compose/infrastructure.yaml deleted file mode 100644 index 9a17a5b3..00000000 --- a/deployments/docker-compose/infrastructure.yaml +++ /dev/null @@ -1,234 +0,0 @@ -version: "3.3" -services: - - ####################################################### - # Rabbitmq - ####################################################### - rabbitmq: - container_name: rabbitmq - image: rabbitmq:management - restart: unless-stopped - ports: - - 5672:5672 - - 15672:15672 - networks: - - booking - - - ####################################################### - # Postgres - ###################################################### - postgres: - image: postgres:latest - container_name: postgres - restart: unless-stopped - ports: - - '5432:5432' - environment: - - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=postgres - command: - - "postgres" - - "-c" - - "wal_level=logical" - - "-c" - - "max_prepared_transactions=10" - networks: - - booking - - - ####################################################### - # SqlServer - ####################################################### - # sql: - # container_name: sql - # image: mcr.microsoft.com/mssql/server - # restart: unless-stopped - # ports: - # - "1433:1433" - # environment: - # SA_PASSWORD: "Password@1234" - # ACCEPT_EULA: "Y" - - - ####################################################### - # Jaeger - ####################################################### - jaeger: - container_name: jaeger - image: jaegertracing/all-in-one - restart: unless-stopped - networks: - - booking - ports: - - 5775:5775/udp - - 5778:5778 - - 6831:6831/udp - - 6832:6832/udp - - 9411:9411 - - 14268:14268 - - 16686:16686 - - - # ####################################################### - # # EventStoreDB - # ####################################################### - # #https://stackoverflow.com/questions/65272764/ports-are-not-available-listen-tcp-0-0-0-0-50070-bind-an-attempt-was-made-to - - eventstore: - container_name: eventstore - image: eventstore/eventstore:latest - restart: unless-stopped - environment: - - EVENTSTORE_CLUSTER_SIZE=1 - - EVENTSTORE_RUN_PROJECTIONS=All - - EVENTSTORE_START_STANDARD_PROJECTIONS=True - - EVENTSTORE_HTTP_PORT=2113 - - EVENTSTORE_INSECURE=True - - EVENTSTORE_ENABLE_ATOM_PUB_OVER_HTTP=True - ports: - - "2113:2113" - networks: - - booking - - - ####################################################### - # Mongo - ####################################################### - mongo: - image: mongo:latest - container_name: mongo - restart: unless-stopped - # environment: - # - MONGO_INITDB_ROOT_USERNAME=root - # - MONGO_INITDB_ROOT_PASSWORD=secret - networks: - - booking - ports: - - 27017:27017 - - - ####################################################### - # Elastic Search - ####################################################### - elasticsearch: - container_name: elasticsearch - image: elasticsearch:7.17.9 - restart: unless-stopped - ports: - - 9200:9200 - volumes: - - elasticsearch-data:/usr/share/elasticsearch/data - environment: - - xpack.monitoring.enabled=true - - xpack.watcher.enabled=false - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - - discovery.type=single-node - networks: - - booking - - ####################################################### - # Kibana - ####################################################### - kibana: - container_name: kibana - image: kibana:7.17.9 - restart: unless-stopped - ports: - - 5601:5601 - depends_on: - - elasticsearch - environment: - - ELASTICSEARCH_URL=http://localhost:9200 - networks: - - booking - - ####################################################### - # Redis - ####################################################### - redis: - image: redis - container_name: redis - restart: unless-stopped - networks: - - booking - ports: - - 6379:6379 - - ####################################################### - # prometheus - ####################################################### - prometheus: - image: prom/prometheus:latest - container_name: prometheus - restart: unless-stopped - ports: - - "9090:9090" - environment: - - TZ=UTC - volumes: - - ./monitoring/prom/prometheus.yml:/etc/prometheus/prometheus.yml - networks: - - booking - - ####################################################### - # grafana - ####################################################### - grafana: - image: grafana/grafana - container_name: grafana - restart: unless-stopped - ports: - - "3000:3000" - volumes: - - ./monitoring/grafana-data/data:/var/lib/grafana - networks: - - booking - - ####################################################### - # node_exporter - ####################################################### - node_exporter: - image: quay.io/prometheus/node-exporter:latest - container_name: node_exporter - restart: unless-stopped - command: - - '--path.rootfs=/host' - pid: host - volumes: - - /proc:/host/proc:ro - - /sys:/host/sys:ro - - /:/rootfs:ro - networks: - - booking - - ####################################################### - # cadvisor - ####################################################### - cadvisor: - image: gcr.io/cadvisor/cadvisor:latest - container_name: cadvisor - restart: unless-stopped - ports: - - "8080:8080" - volumes: - - /:/rootfs:ro - - /var/run:/var/run:ro - - /sys:/sys:ro - - /var/lib/docker/:/var/lib/docker:ro - - /dev/disk/:/dev/disk:ro - devices: - - /dev/kmsg - networks: - - booking - - -networks: - booking: - -volumes: - elasticsearch-data: - - - - diff --git a/deployments/docker-compose/monitoring/prom/prometheus.yml b/deployments/docker-compose/monitoring/prom/prometheus.yml deleted file mode 100644 index 6aaff5bf..00000000 --- a/deployments/docker-compose/monitoring/prom/prometheus.yml +++ /dev/null @@ -1,69 +0,0 @@ -global: - scrape_interval: 15s - scrape_timeout: 10s - evaluation_interval: 15s -alerting: - alertmanagers: - - scheme: http - timeout: 10s - api_version: v1 - static_configs: - - targets: [] -scrape_configs: -- job_name: prometheus - honor_timestamps: true - scrape_interval: 15s - scrape_timeout: 10s - metrics_path: /metrics - scheme: http - static_configs: - - targets: - - localhost:9090 -- job_name: cadvisor - static_configs: - - targets: - - cadvisor:8080 -- job_name: node_exporter - static_configs: - - targets: - - node_exporter:9100 -- job_name: flight - scrape_interval: 15s - scrape_timeout: 10s - metrics_path: /metrics - scheme: http - static_configs: - - targets: - - host.docker.internal:5004 -- job_name: identity - scrape_interval: 15s - scrape_timeout: 10s - metrics_path: /metrics - scheme: http - static_configs: - - targets: - - host.docker.internal:6005 -- job_name: passenger - scrape_interval: 15s - scrape_timeout: 10s - metrics_path: /metrics - scheme: http - static_configs: - - targets: - - host.docker.internal:6012 -- job_name: booking - scrape_interval: 15s - scrape_timeout: 10s - metrics_path: /metrics - scheme: http - static_configs: - - targets: - - host.docker.internal:6010 -- job_name: gateway - scrape_interval: 15s - scrape_timeout: 10s - metrics_path: /metrics - scheme: https - static_configs: - - targets: - - host.docker.internal:5000 \ No newline at end of file diff --git a/deployments/kubernetes/booking-microservices.yml b/deployments/kubernetes/booking-microservices.yml index 5a1ac92a..68aa5b29 100644 --- a/deployments/kubernetes/booking-microservices.yml +++ b/deployments/kubernetes/booking-microservices.yml @@ -24,89 +24,95 @@ spec: apiVersion: apps/v1 kind: Deployment metadata: - annotations: - kompose.cmd: kompose convert -f infrastracture.yaml - kompose.version: 1.26.0 (40646f47) - creationTimestamp: null - labels: - io.kompose.service: elasticsearch name: elasticsearch spec: replicas: 1 selector: matchLabels: - io.kompose.service: elasticsearch - strategy: - type: Recreate + app: elasticsearch template: metadata: - annotations: - kompose.cmd: kompose convert -f infrastracture.yaml - kompose.version: 1.26.0 (40646f47) - creationTimestamp: null labels: - io.kompose.network/booking: "true" - io.kompose.service: elasticsearch + app: elasticsearch spec: containers: - - env: - - name: ES_JAVA_OPTS - value: -Xms512m -Xmx512m - - name: discovery.type - value: single-node - - name: xpack.monitoring.enabled - value: "true" - - name: xpack.watcher.enabled - value: "false" - image: elasticsearch:7.17.9 - name: elasticsearch - ports: - - containerPort: 9200 - resources: {} - volumeMounts: - - mountPath: /usr/share/elasticsearch/data - name: elasticsearch-data - restartPolicy: Always + - name: elasticsearch + image: docker.elastic.co/elasticsearch/elasticsearch:8.17.0 + env: + - name: discovery.type + value: "single-node" + - name: cluster.name + value: "docker-cluster" + - name: node.name + value: "docker-node" + - name: ES_JAVA_OPTS + value: "-Xms512m -Xmx512m" + - name: xpack.security.enabled + value: "false" + - name: xpack.security.http.ssl.enabled + value: "false" + - name: xpack.security.transport.ssl.enabled + value: "false" + - name: network.host + value: "0.0.0.0" + - name: http.port + value: "9200" + - name: transport.host + value: "localhost" + - name: bootstrap.memory_lock + value: "true" + - name: cluster.routing.allocation.disk.threshold_enabled + value: "false" + ports: + - containerPort: 9200 + - containerPort: 9300 + volumeMounts: + - mountPath: /usr/share/elasticsearch/data + name: elastic-data volumes: - - name: elasticsearch-data - persistentVolumeClaim: - claimName: elasticsearch-data -status: {} + - name: elastic-data + persistentVolumeClaim: + claimName: elasticsearch-pvc +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: elasticsearch-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi --- apiVersion: v1 kind: Service metadata: - annotations: - kompose.cmd: kompose convert -f infrastracture.yaml - kompose.version: 1.26.0 (40646f47) - creationTimestamp: null - labels: - io.kompose.service: elasticsearch name: elasticsearch spec: - ports: - - name: "9200" - port: 9200 - targetPort: 9200 selector: - io.kompose.service: elasticsearch -status: - loadBalancer: {} + app: elasticsearch + ports: + - port: 9200 + targetPort: 9200 + - port: 9300 + targetPort: 9300 + type: ClusterIP --- apiVersion: v1 -kind: PersistentVolumeClaim +kind: PersistentVolume metadata: - creationTimestamp: null + name: elasticsearch-pv labels: - io.kompose.service: elasticsearch-data - name: elasticsearch-data + type: local spec: + storageClassName: manual + capacity: + storage: 10Gi accessModes: - ReadWriteOnce - resources: - requests: - storage: 100Mi -status: {} + hostPath: + path: "/mnt/data" --- ####################################################### # Kibana @@ -114,847 +120,824 @@ status: {} apiVersion: apps/v1 kind: Deployment metadata: - annotations: - kompose.cmd: kompose convert -f infrastracture.yaml - kompose.version: 1.26.0 (40646f47) - creationTimestamp: null - labels: - io.kompose.service: kibana name: kibana spec: replicas: 1 selector: matchLabels: - io.kompose.service: kibana - strategy: {} + app: kibana template: metadata: - annotations: - kompose.cmd: kompose convert -f infrastracture.yaml - kompose.version: 1.26.0 (40646f47) - creationTimestamp: null labels: - io.kompose.network/booking: "true" - io.kompose.service: kibana + app: kibana spec: containers: - - env: - - name: ELASTICSEARCH_URL - value: elasticsearch:9200 - image: kibana:7.17.9 - name: kibana - ports: - - containerPort: 5601 - resources: {} - restartPolicy: Always -status: {} + - name: kibana + image: docker.elastic.co/kibana/kibana:8.17.0 + env: + - name: ELASTICSEARCH_HOSTS + value: "http://elasticsearch:9200" + ports: + - containerPort: 5601 + resources: + requests: + memory: "512Mi" + cpu: "500m" + limits: + memory: "1Gi" + cpu: "1" --- apiVersion: v1 kind: Service metadata: - annotations: - kompose.cmd: kompose convert -f infrastracture.yaml - kompose.version: 1.26.0 (40646f47) - creationTimestamp: null - labels: - io.kompose.service: kibana name: kibana spec: - ports: - - name: "5601" - port: 5601 - targetPort: 5601 selector: - io.kompose.service: kibana -status: - loadBalancer: {} + app: kibana + ports: + - port: 5601 + targetPort: 5601 + type: ClusterIP --- ####################################################### -# Event Store +# Tempo ####################################################### apiVersion: apps/v1 kind: Deployment metadata: - annotations: - kompose.cmd: C:\ProgramData\chocolatey\lib\kubernetes-kompose\tools\kompose.exe convert -f infrastructure.yaml - kompose.version: 1.34.0 (cbf2835db) - labels: - io.kompose.service: eventstore - name: eventstore + name: tempo spec: replicas: 1 selector: matchLabels: - io.kompose.service: eventstore + app: tempo template: metadata: - annotations: - kompose.cmd: C:\ProgramData\chocolatey\lib\kubernetes-kompose\tools\kompose.exe convert -f infrastructure.yaml - kompose.version: 1.34.0 (cbf2835db) labels: - io.kompose.service: eventstore + app: tempo spec: containers: - - env: - - name: EVENTSTORE_CLUSTER_SIZE - value: "1" - - name: EVENTSTORE_ENABLE_ATOM_PUB_OVER_HTTP - value: "True" - - name: EVENTSTORE_HTTP_PORT - value: "2113" - - name: EVENTSTORE_INSECURE - value: "True" - - name: EVENTSTORE_RUN_PROJECTIONS - value: All - - name: EVENTSTORE_START_STANDARD_PROJECTIONS - value: "True" - image: eventstore/eventstore:latest - name: eventstore - ports: - - containerPort: 2113 - protocol: TCP - restartPolicy: Always + - name: tempo + image: grafana/tempo:latest + args: + - "-config.file=/etc/tempo.yaml" + ports: + - containerPort: 3200 + - containerPort: 4317 + - containerPort: 4318 + volumeMounts: + - mountPath: /etc/tempo.yaml + name: tempo-config + subPath: tempo.yaml + volumes: + - name: tempo-config + configMap: + name: tempo-config +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: tempo-config +data: + tempo.yaml: | + # Your Tempo configuration here --- apiVersion: v1 kind: Service metadata: - annotations: - kompose.cmd: C:\ProgramData\chocolatey\lib\kubernetes-kompose\tools\kompose.exe convert -f infrastructure.yaml - kompose.version: 1.34.0 (cbf2835db) - labels: - io.kompose.service: eventstore - name: eventstore + name: tempo spec: - ports: - - name: "2113" - port: 2113 - targetPort: 2113 selector: - io.kompose.service: eventstore + app: tempo + ports: + - port: 3200 + targetPort: 3200 + - port: 4317 + targetPort: 4317 + - port: 4318 + targetPort: 4318 + type: ClusterIP --- -# ####################################################### -# # Jaeger -# ####################################################### +####################################################### +# Looki +####################################################### apiVersion: apps/v1 kind: Deployment metadata: - annotations: - kompose.cmd: C:\ProgramData\chocolatey\lib\kubernetes-kompose\tools\kompose.exe -f infrastracture.yaml convert - kompose.version: 1.28.0 (c4137012e) - creationTimestamp: null - labels: - io.kompose.service: jaeger - name: jaeger + name: loki spec: replicas: 1 selector: matchLabels: - io.kompose.service: jaeger - strategy: {} + app: loki template: metadata: - annotations: - kompose.cmd: C:\ProgramData\chocolatey\lib\kubernetes-kompose\tools\kompose.exe -f infrastracture.yaml convert - kompose.version: 1.28.0 (c4137012e) - creationTimestamp: null labels: - io.kompose.network/booking: "true" - io.kompose.service: jaeger + app: loki spec: containers: - - image: jaegertracing/all-in-one - name: jaeger - ports: - - containerPort: 5775 - protocol: UDP - - containerPort: 5778 - - containerPort: 6831 - protocol: UDP - - containerPort: 6832 - protocol: UDP - - containerPort: 9411 - - containerPort: 14268 - - containerPort: 16686 - resources: {} - restartPolicy: Always -status: {} + - name: loki + image: grafana/loki:latest + args: + - "-config.file=/etc/loki/local-config.yaml" + ports: + - containerPort: 3100 + volumeMounts: + - mountPath: /etc/loki/local-config.yaml + name: loki-config + subPath: local-config.yaml + volumes: + - name: loki-config + configMap: + name: loki-config +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: loki-config +data: + local-config.yaml: | + # Your Loki configuration here --- apiVersion: v1 kind: Service metadata: - annotations: - kompose.cmd: C:\ProgramData\chocolatey\lib\kubernetes-kompose\tools\kompose.exe -f infrastracture.yaml convert - kompose.version: 1.28.0 (c4137012e) - creationTimestamp: null - labels: - io.kompose.service: jaeger - name: jaeger + name: loki spec: - ports: - - name: "5775" - port: 5775 - protocol: UDP - targetPort: 5775 - - name: "5778" - port: 5778 - targetPort: 5778 - - name: "6831" - port: 6831 - protocol: UDP - targetPort: 6831 - - name: "6832" - port: 6832 - protocol: UDP - targetPort: 6832 - - name: "9411" - port: 9411 - targetPort: 9411 - - name: "14268" - port: 14268 - targetPort: 14268 - - name: "16686" - port: 16686 - targetPort: 16686 selector: - io.kompose.service: jaeger -status: - loadBalancer: {} + app: loki + ports: + - port: 3100 + targetPort: 3100 + type: ClusterIP --- ####################################################### -# Prometheus +# Event Store ####################################################### apiVersion: apps/v1 kind: Deployment metadata: - annotations: - kompose.cmd: C:\ProgramData\chocolatey\lib\kubernetes-kompose\tools\kompose.exe convert -f infrastracture.yaml - kompose.version: 1.28.0 (c4137012e) - creationTimestamp: null - labels: - io.kompose.service: prometheus - name: prometheus + name: eventstore spec: replicas: 1 selector: matchLabels: - io.kompose.service: prometheus - strategy: - type: Recreate + app: eventstore template: metadata: - annotations: - kompose.cmd: C:\ProgramData\chocolatey\lib\kubernetes-kompose\tools\kompose.exe convert -f infrastracture.yaml - kompose.version: 1.28.0 (c4137012e) - creationTimestamp: null labels: - io.kompose.network/newfolder-booking: "true" - io.kompose.service: prometheus + app: eventstore spec: containers: - - env: - - name: TZ - value: UTC - image: prom/prometheus:latest - name: prometheus - ports: - - containerPort: 9090 - resources: {} - volumeMounts: - - mountPath: /etc/prometheus/prometheus.yml - name: prometheus-claim0 - restartPolicy: Always - volumes: - - name: prometheus-claim0 - persistentVolumeClaim: - claimName: prometheus-claim0 -status: {} + - name: eventstore + image: eventstore/eventstore:latest + env: + - name: EVENTSTORE_CLUSTER_SIZE + value: "1" + - name: EVENTSTORE_RUN_PROJECTIONS + value: "All" + - name: EVENTSTORE_START_STANDARD_PROJECTIONS + value: "True" + - name: EVENTSTORE_HTTP_PORT + value: "2113" + - name: EVENTSTORE_INSECURE + value: "True" + - name: EVENTSTORE_ENABLE_ATOM_PUB_OVER_HTTP + value: "True" + ports: + - containerPort: 2113 + resources: + requests: + memory: "512Mi" + cpu: "500m" + limits: + memory: "1Gi" + cpu: "1" --- apiVersion: v1 kind: Service metadata: - annotations: - kompose.cmd: C:\ProgramData\chocolatey\lib\kubernetes-kompose\tools\kompose.exe convert -f infrastracture.yaml - kompose.version: 1.28.0 (c4137012e) - creationTimestamp: null - labels: - io.kompose.service: prometheus - name: prometheus + name: eventstore spec: + selector: + app: eventstore ports: - - name: "9090" - port: 9090 - targetPort: 9090 + - port: 2113 + targetPort: 2113 + type: ClusterIP +--- +####################################################### +# Jaeger +####################################################### +apiVersion: apps/v1 +kind: Deployment +metadata: + name: jaeger +spec: + replicas: 1 selector: - io.kompose.service: prometheus -status: - loadBalancer: {} + matchLabels: + app: jaeger + template: + metadata: + labels: + app: jaeger + spec: + containers: + - name: jaeger + image: jaegertracing/all-in-one:latest + ports: + - containerPort: 6831 + protocol: UDP + - containerPort: 16686 + - containerPort: 14268 + - containerPort: 4317 + - containerPort: 4318 + resources: + requests: + memory: "512Mi" + cpu: "500m" + limits: + memory: "1Gi" + cpu: "1" --- apiVersion: v1 -kind: PersistentVolumeClaim +kind: Service metadata: - creationTimestamp: null - labels: - io.kompose.service: prometheus-claim0 - name: prometheus-claim0 + name: jaeger spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 100Mi -status: {} + selector: + app: jaeger + ports: + - port: 6831 + targetPort: 6831 + protocol: UDP + - port: 16686 + targetPort: 16686 + - port: 14268 + targetPort: 14268 + - port: 4317 + targetPort: 4317 + - port: 4318 + targetPort: 4318 + type: ClusterIP --- ####################################################### -# Grafana +# Zipkin ####################################################### apiVersion: apps/v1 kind: Deployment metadata: - annotations: - kompose.cmd: C:\ProgramData\chocolatey\lib\kubernetes-kompose\tools\kompose.exe convert -f infrastracture.yaml - kompose.version: 1.28.0 (c4137012e) - creationTimestamp: null - labels: - io.kompose.service: grafana - name: grafana + name: zipkin spec: replicas: 1 selector: matchLabels: - io.kompose.service: grafana - strategy: - type: Recreate + app: zipkin template: metadata: - annotations: - kompose.cmd: C:\ProgramData\chocolatey\lib\kubernetes-kompose\tools\kompose.exe convert -f infrastracture.yaml - kompose.version: 1.28.0 (c4137012e) - creationTimestamp: null labels: - io.kompose.network/newfolder-booking: "true" - io.kompose.service: grafana + app: zipkin spec: containers: - - image: grafana/grafana - name: grafana - ports: - - containerPort: 3000 - resources: {} - volumeMounts: - - mountPath: /var/lib/grafana - name: grafana-claim0 - restartPolicy: Always - volumes: - - name: grafana-claim0 - persistentVolumeClaim: - claimName: grafana-claim0 -status: {} + - name: zipkin + image: openzipkin/zipkin:latest + ports: + - containerPort: 9411 + resources: + requests: + memory: "512Mi" + cpu: "500m" + limits: + memory: "1Gi" + cpu: "1" --- apiVersion: v1 kind: Service metadata: - annotations: - kompose.cmd: C:\ProgramData\chocolatey\lib\kubernetes-kompose\tools\kompose.exe convert -f infrastracture.yaml - kompose.version: 1.28.0 (c4137012e) - creationTimestamp: null - labels: - io.kompose.service: grafana - name: grafana + name: zipkin spec: + selector: + app: zipkin ports: - - name: "3000" - port: 3000 - targetPort: 3000 + - port: 9411 + targetPort: 9411 + type: ClusterIP + +--- +####################################################### +# OpenTelemetry Collector +####################################################### +apiVersion: apps/v1 +kind: Deployment +metadata: + name: otel-collector +spec: + replicas: 1 selector: - io.kompose.service: grafana -status: - loadBalancer: {} + matchLabels: + app: otel-collector + template: + metadata: + labels: + app: otel-collector + spec: + containers: + - name: otel-collector + image: otel/opentelemetry-collector-contrib:latest + args: ["--config=/etc/otelcol-contrib/config.yaml"] + ports: + - containerPort: 11888 + - containerPort: 8888 + - containerPort: 8889 + - containerPort: 13133 + - containerPort: 4317 + - containerPort: 4318 + - containerPort: 55679 + volumeMounts: + - mountPath: /etc/otelcol-contrib/config.yaml + name: otel-config + subPath: config.yaml + volumes: + - name: otel-config + configMap: + name: otel-collector-config --- apiVersion: v1 -kind: PersistentVolumeClaim +kind: ConfigMap metadata: - creationTimestamp: null - labels: - io.kompose.service: grafana-claim0 - name: grafana-claim0 + name: otel-collector-config +data: + config.yaml: | + # Your OpenTelemetry Collector configuration here +--- +apiVersion: v1 +kind: Service +metadata: + name: otel-collector spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 100Mi -status: {} + selector: + app: otel-collector + ports: + - port: 11888 + targetPort: 11888 + - port: 8888 + targetPort: 8888 + - port: 8889 + targetPort: 8889 + - port: 13133 + targetPort: 13133 + - port: 4317 + targetPort: 4317 + - port: 4318 + targetPort: 4318 + - port: 55679 + targetPort: 55679 + type: ClusterIP --- -# ####################################################### -# # Node Exporter -# ####################################################### +####################################################### +# Prometheus +####################################################### apiVersion: apps/v1 kind: Deployment metadata: - annotations: - kompose.cmd: C:\ProgramData\chocolatey\lib\kubernetes-kompose\tools\kompose.exe convert -f infrastructure.yaml - kompose.version: 1.34.0 (cbf2835db) - labels: - io.kompose.service: node-exporter - name: node-exporter + name: prometheus spec: replicas: 1 selector: matchLabels: - io.kompose.service: node-exporter - strategy: - type: Recreate + app: prometheus template: metadata: - annotations: - kompose.cmd: C:\ProgramData\chocolatey\lib\kubernetes-kompose\tools\kompose.exe convert -f infrastructure.yaml - kompose.version: 1.34.0 (cbf2835db) labels: - io.kompose.service: node-exporter + app: prometheus spec: containers: - - args: - - --path.rootfs=/host - image: quay.io/prometheus/node-exporter:latest - name: node-exporter - volumeMounts: - - mountPath: /host/proc - name: node-exporter-claim0 - readOnly: true - - mountPath: /host/sys - name: node-exporter-claim1 - readOnly: true - restartPolicy: Always + - name: prometheus + image: prom/prometheus:latest + args: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--web.console.libraries=/usr/share/prometheus/console_libraries" + - "--web.console.templates=/usr/share/prometheus/consoles" + - "--web.enable-remote-write-receiver" + ports: + - containerPort: 9090 + volumeMounts: + - mountPath: /etc/prometheus/prometheus.yml + name: prometheus-config + subPath: prometheus.yml volumes: - - name: node-exporter-claim0 - persistentVolumeClaim: - claimName: node-exporter-claim0 - readOnly: true - - name: node-exporter-claim1 - persistentVolumeClaim: - claimName: node-exporter-claim1 - readOnly: true + - name: prometheus-config + configMap: + name: prometheus-config --- apiVersion: v1 -kind: PersistentVolumeClaim +kind: ConfigMap metadata: - labels: - io.kompose.service: node-exporter-claim0 - name: node-exporter-claim0 -spec: - accessModes: - - ReadOnlyMany - resources: - requests: - storage: 100Mi + name: prometheus-config +data: + prometheus.yml: | + # Your Prometheus configuration here --- apiVersion: v1 -kind: PersistentVolumeClaim +kind: Service metadata: - labels: - io.kompose.service: node-exporter-claim1 - name: node-exporter-claim1 + name: prometheus spec: - accessModes: - - ReadOnlyMany - resources: - requests: - storage: 100Mi + selector: + app: prometheus + ports: + - port: 9090 + targetPort: 9090 + type: ClusterIP --- -# ####################################################### -# # Cadvisor -# ####################################################### +####################################################### +# Grafana +####################################################### apiVersion: apps/v1 kind: Deployment metadata: - annotations: - kompose.cmd: C:\ProgramData\chocolatey\lib\kubernetes-kompose\tools\kompose.exe convert -f infrastracture.yaml - kompose.version: 1.28.0 (c4137012e) - creationTimestamp: null - labels: - io.kompose.service: cadvisor - name: cadvisor + name: grafana spec: replicas: 1 selector: matchLabels: - io.kompose.service: cadvisor - strategy: - type: Recreate + app: grafana template: metadata: - annotations: - kompose.cmd: C:\ProgramData\chocolatey\lib\kubernetes-kompose\tools\kompose.exe convert -f infrastracture.yaml - kompose.version: 1.28.0 (c4137012e) - creationTimestamp: null labels: - io.kompose.network/newfolder-booking: "true" - io.kompose.service: cadvisor + app: grafana spec: containers: - - image: gcr.io/cadvisor/cadvisor:latest - name: cadvisor - ports: - - containerPort: 8080 - resources: {} - volumeMounts: - - mountPath: /rootfs - name: cadvisor-claim0 - readOnly: true - - mountPath: /var/run - name: cadvisor-claim1 - readOnly: true - - mountPath: /sys - name: cadvisor-claim2 - readOnly: true - - mountPath: /var/lib/docker - name: cadvisor-claim3 - readOnly: true - - mountPath: /dev/disk - name: cadvisor-claim4 - readOnly: true - restartPolicy: Always + - name: grafana + image: grafana/grafana:latest + env: + - name: GF_INSTALL_PLUGINS + value: "grafana-clock-panel,grafana-simple-json-datasource" + - name: GF_SECURITY_ADMIN_USER + value: "admin" + - name: GF_SECURITY_ADMIN_PASSWORD + value: "admin" + - name: GF_FEATURE_TOGGLES_ENABLE + value: "traceqlEditor" + ports: + - containerPort: 3000 + volumeMounts: + - mountPath: /etc/grafana/provisioning + name: grafana-provisioning + - mountPath: /var/lib/grafana/dashboards + name: grafana-dashboards volumes: - - name: cadvisor-claim0 - persistentVolumeClaim: - claimName: cadvisor-claim0 - readOnly: true - - name: cadvisor-claim1 - persistentVolumeClaim: - claimName: cadvisor-claim1 - readOnly: true - - name: cadvisor-claim2 - persistentVolumeClaim: - claimName: cadvisor-claim2 - readOnly: true - - name: cadvisor-claim3 - persistentVolumeClaim: - claimName: cadvisor-claim3 - readOnly: true - - name: cadvisor-claim4 - persistentVolumeClaim: - claimName: cadvisor-claim4 - readOnly: true -status: {} + - name: grafana-provisioning + configMap: + name: grafana-provisioning + - name: grafana-dashboards + configMap: + name: grafana-dashboards --- apiVersion: v1 -kind: Service +kind: ConfigMap metadata: - annotations: - kompose.cmd: C:\ProgramData\chocolatey\lib\kubernetes-kompose\tools\kompose.exe convert -f infrastracture.yaml - kompose.version: 1.28.0 (c4137012e) - creationTimestamp: null - labels: - io.kompose.service: cadvisor - name: cadvisor -spec: - ports: - - name: "8080" - port: 8080 - targetPort: 8080 - selector: - io.kompose.service: cadvisor -status: - loadBalancer: {} + name: grafana-provisioning +data: + # Your Grafana provisioning configuration here --- apiVersion: v1 -kind: PersistentVolumeClaim +kind: ConfigMap metadata: - creationTimestamp: null - labels: - io.kompose.service: cadvisor-claim0 - name: cadvisor-claim0 -spec: - accessModes: - - ReadOnlyMany - resources: - requests: - storage: 100Mi -status: {} + name: grafana-dashboards +data: + # Your Grafana dashboards configuration here --- apiVersion: v1 -kind: PersistentVolumeClaim +kind: Service metadata: - creationTimestamp: null - labels: - io.kompose.service: cadvisor-claim1 - name: cadvisor-claim1 + name: grafana spec: - accessModes: - - ReadOnlyMany - resources: - requests: - storage: 100Mi -status: {} + selector: + app: grafana + ports: + - port: 3000 + targetPort: 3000 + type: ClusterIP --- -apiVersion: v1 -kind: PersistentVolumeClaim +####################################################### +# Node Exporter +####################################################### +apiVersion: apps/v1 +kind: Deployment metadata: - creationTimestamp: null - labels: - io.kompose.service: cadvisor-claim2 - name: cadvisor-claim2 + name: node-exporter spec: - accessModes: - - ReadOnlyMany - resources: - requests: - storage: 100Mi -status: {} + replicas: 1 + selector: + matchLabels: + app: node-exporter + template: + metadata: + labels: + app: node-exporter + spec: + containers: + - name: node-exporter + image: prom/node-exporter:latest + args: + - "--path.procfs=/host/proc" + - "--path.rootfs=/rootfs" + - "--path.sysfs=/host/sys" + ports: + - containerPort: 9100 + volumeMounts: + - mountPath: /host/proc + name: proc + readOnly: true + - mountPath: /host/sys + name: sys + readOnly: true + - mountPath: /rootfs + name: rootfs + readOnly: true + volumes: + - name: proc + hostPath: + path: /proc + - name: sys + hostPath: + path: /sys + - name: rootfs + hostPath: + path: / --- apiVersion: v1 -kind: PersistentVolumeClaim +kind: Service metadata: - creationTimestamp: null - labels: - io.kompose.service: cadvisor-claim3 - name: cadvisor-claim3 + name: node-exporter spec: - accessModes: - - ReadOnlyMany - resources: - requests: - storage: 100Mi -status: {} + selector: + app: node-exporter + ports: + - port: 9100 + targetPort: 9100 + type: ClusterIP +--- +####################################################### +# Cadvisor +####################################################### +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cadvisor +spec: + replicas: 1 + selector: + matchLabels: + app: cadvisor + template: + metadata: + labels: + app: cadvisor + spec: + containers: + - name: cadvisor + image: gcr.io/cadvisor/cadvisor:latest + ports: + - containerPort: 8080 + volumeMounts: + - mountPath: /rootfs + name: rootfs + readOnly: true + - mountPath: /var/run + name: var-run + readOnly: true + - mountPath: /sys + name: sys + readOnly: true + - mountPath: /var/lib/docker + name: var-lib-docker + readOnly: true + - mountPath: /dev/disk + name: dev-disk + readOnly: true + volumes: + - name: rootfs + hostPath: + path: / + - name: var-run + hostPath: + path: /var/run + - name: sys + hostPath: + path: /sys + - name: var-lib-docker + hostPath: + path: /var/lib/docker + - name: dev-disk + hostPath: + path: /dev/disk --- apiVersion: v1 -kind: PersistentVolumeClaim +kind: Service metadata: - creationTimestamp: null - labels: - io.kompose.service: cadvisor-claim4 - name: cadvisor-claim4 + name: cadvisor spec: - accessModes: - - ReadOnlyMany - resources: - requests: - storage: 100Mi -status: {} + selector: + app: cadvisor + ports: + - port: 8080 + targetPort: 8080 + type: ClusterIP --- -# ####################################################### -# # Mongo -# ####################################################### +####################################################### +# Mongo +####################################################### apiVersion: apps/v1 kind: Deployment metadata: - annotations: - kompose.cmd: C:\ProgramData\chocolatey\lib\kubernetes-kompose\tools\kompose.exe -f infrastracture.yaml convert - kompose.version: 1.28.0 (c4137012e) - creationTimestamp: null - labels: - io.kompose.service: mongo name: mongo spec: replicas: 1 selector: matchLabels: - io.kompose.service: mongo - strategy: {} + app: mongo template: metadata: - annotations: - kompose.cmd: C:\ProgramData\chocolatey\lib\kubernetes-kompose\tools\kompose.exe -f infrastracture.yaml convert - kompose.version: 1.28.0 (c4137012e) - creationTimestamp: null labels: - io.kompose.network/booking: "true" - io.kompose.service: mongo + app: mongo spec: containers: - - image: mongo:latest - name: mongo - ports: - - containerPort: 27017 - resources: {} - restartPolicy: Always -status: {} + - name: mongo + image: mongo:latest + ports: + - containerPort: 27017 + resources: + requests: + memory: "512Mi" + cpu: "500m" + limits: + memory: "1Gi" + cpu: "1" --- apiVersion: v1 kind: Service metadata: - annotations: - kompose.cmd: C:\ProgramData\chocolatey\lib\kubernetes-kompose\tools\kompose.exe -f infrastracture.yaml convert - kompose.version: 1.28.0 (c4137012e) - creationTimestamp: null - labels: - io.kompose.service: mongo name: mongo spec: - ports: - - name: "27017" - port: 27017 - targetPort: 27017 selector: - io.kompose.service: mongo -status: - loadBalancer: {} + app: mongo + ports: + - port: 27017 + targetPort: 27017 + type: ClusterIP --- -# ####################################################### -# # Postgres -# ####################################################### -apiVersion: v1 -kind: Pod +####################################################### +# Postgres +####################################################### +apiVersion: apps/v1 +kind: Deployment metadata: - labels: - app: postgres name: postgres spec: - containers: - - args: - - postgres - - -c - - wal_level=logical - - -c - - max_prepared_transactions=10 - env: - - name: POSTGRES_PASSWORD - value: postgres + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: postgres:latest + env: - name: POSTGRES_USER value: postgres - image: postgres:latest - name: postgres - ports: + - name: POSTGRES_PASSWORD + value: postgres + ports: - containerPort: 5432 - restartPolicy: Always + volumeMounts: + - mountPath: /var/lib/postgresql/data + name: postgres-data + volumes: + - name: postgres-data + persistentVolumeClaim: + claimName: postgres-pvc +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgres-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi --- apiVersion: v1 kind: Service metadata: - labels: - app: postgres name: postgres spec: - ports: - - name: "5432" - port: 5432 - targetPort: 5432 selector: app: postgres + ports: + - port: 5432 + targetPort: 5432 + type: ClusterIP --- -# ####################################################### -# # Rabbitmq -# ####################################################### +####################################################### +# Rabbitmq +####################################################### apiVersion: apps/v1 kind: Deployment metadata: - annotations: - kompose.cmd: C:\ProgramData\chocolatey\lib\kubernetes-kompose\tools\kompose.exe -f infrastracture.yaml convert - kompose.version: 1.28.0 (c4137012e) - creationTimestamp: null - labels: - io.kompose.service: rabbitmq name: rabbitmq spec: replicas: 1 selector: matchLabels: - io.kompose.service: rabbitmq - strategy: {} + app: rabbitmq template: metadata: - annotations: - kompose.cmd: C:\ProgramData\chocolatey\lib\kubernetes-kompose\tools\kompose.exe -f infrastracture.yaml convert - kompose.version: 1.28.0 (c4137012e) - creationTimestamp: null labels: - io.kompose.network/booking: "true" - io.kompose.service: rabbitmq + app: rabbitmq spec: containers: - - image: rabbitmq:management - name: rabbitmq - ports: - - containerPort: 5672 - - containerPort: 15672 - resources: {} - restartPolicy: Always -status: {} + - name: rabbitmq + image: rabbitmq:management + ports: + - containerPort: 5672 + - containerPort: 15672 + resources: + requests: + memory: "512Mi" + cpu: "500m" + limits: + memory: "1Gi" + cpu: "1" --- apiVersion: v1 kind: Service metadata: - annotations: - kompose.cmd: C:\ProgramData\chocolatey\lib\kubernetes-kompose\tools\kompose.exe -f infrastracture.yaml convert - kompose.version: 1.28.0 (c4137012e) - creationTimestamp: null - labels: - io.kompose.service: rabbitmq name: rabbitmq spec: - ports: - - name: "5672" - port: 5672 - targetPort: 5672 - - name: "15672" - port: 15672 - targetPort: 15672 selector: - io.kompose.service: rabbitmq -status: - loadBalancer: {} + app: rabbitmq + ports: + - port: 5672 + targetPort: 5672 + - port: 15672 + targetPort: 15672 + type: ClusterIP --- -# ####################################################### -# # Redis -# ####################################################### +####################################################### +# Redis +####################################################### apiVersion: apps/v1 kind: Deployment metadata: - annotations: - kompose.cmd: C:\ProgramData\chocolatey\lib\kubernetes-kompose\tools\kompose.exe -f infrastracture.yaml convert - kompose.version: 1.28.0 (c4137012e) - creationTimestamp: null - labels: - io.kompose.service: redis name: redis spec: replicas: 1 selector: matchLabels: - io.kompose.service: redis - strategy: {} + app: redis template: metadata: - annotations: - kompose.cmd: C:\ProgramData\chocolatey\lib\kubernetes-kompose\tools\kompose.exe -f infrastracture.yaml convert - kompose.version: 1.28.0 (c4137012e) - creationTimestamp: null labels: - io.kompose.network/booking: "true" - io.kompose.service: redis + app: redis spec: containers: - - image: redis - name: redis - ports: - - containerPort: 6379 - resources: {} - restartPolicy: Always -status: {} + - name: redis + image: redis + ports: + - containerPort: 6379 + resources: + requests: + memory: "512Mi" + cpu: "500m" + limits: + memory: "1Gi" + cpu: "1" --- apiVersion: v1 kind: Service metadata: - annotations: - kompose.cmd: C:\ProgramData\chocolatey\lib\kubernetes-kompose\tools\kompose.exe -f infrastracture.yaml convert - kompose.version: 1.28.0 (c4137012e) - creationTimestamp: null - labels: - io.kompose.service: redis name: redis spec: - ports: - - name: "6379" - port: 6379 - targetPort: 6379 selector: - io.kompose.service: redis -status: - loadBalancer: {} + app: redis + ports: + - port: 6379 + targetPort: 6379 + type: ClusterIP --- -# ####################################################### +####################################################### # ConfigMap AppSettings -# ####################################################### +####################################################### apiVersion: v1 kind: ConfigMap metadata: @@ -966,9 +949,9 @@ data: } #ref: https://www.mrjamiebowman.com/software-development/dotnet/kubernetes-configmaps-with-net-core/ --- -# ####################################################### +####################################################### # Flight -# ####################################################### +####################################################### apiVersion: apps/v1 kind: Deployment metadata: @@ -1017,9 +1000,9 @@ spec: targetPort: 80 type: ClusterIP --- -# ####################################################### +####################################################### # Identity -# ####################################################### +####################################################### apiVersion: apps/v1 kind: Deployment metadata: @@ -1068,9 +1051,9 @@ spec: targetPort: 80 type: ClusterIP --- -# ####################################################### +####################################################### # Booking -# ####################################################### +####################################################### apiVersion: apps/v1 kind: Deployment metadata: @@ -1119,9 +1102,9 @@ spec: targetPort: 80 type: ClusterIP --- -# ####################################################### +####################################################### # Passenger -# ####################################################### +####################################################### apiVersion: apps/v1 kind: Deployment metadata: @@ -1156,9 +1139,9 @@ spec: configMap: name: appsettings --- -# ####################################################### +####################################################### # Ingress Controller -# ####################################################### +####################################################### apiVersion: v1 kind: Service metadata: diff --git a/src/BuildingBlocks/BuildingBlocks.csproj b/src/BuildingBlocks/BuildingBlocks.csproj index 2a34911d..7f8b487c 100644 --- a/src/BuildingBlocks/BuildingBlocks.csproj +++ b/src/BuildingBlocks/BuildingBlocks.csproj @@ -29,8 +29,6 @@ - - @@ -50,10 +48,8 @@ - - @@ -68,8 +64,8 @@ - - + + @@ -85,14 +81,18 @@ - - - - - - - - + + + + + + + + + + + + diff --git a/src/BuildingBlocks/HealthCheck/Extensions.cs b/src/BuildingBlocks/HealthCheck/Extensions.cs index bd8623ed..137ed4e9 100644 --- a/src/BuildingBlocks/HealthCheck/Extensions.cs +++ b/src/BuildingBlocks/HealthCheck/Extensions.cs @@ -2,6 +2,7 @@ using BuildingBlocks.Logging; using BuildingBlocks.MassTransit; using BuildingBlocks.Mongo; +using BuildingBlocks.OpenTelemetryCollector; using BuildingBlocks.Web; using HealthChecks.UI.Client; using Microsoft.AspNetCore.Builder; @@ -25,13 +26,11 @@ public static IServiceCollection AddCustomHealthCheck(this IServiceCollection se var postgresOptions = services.GetOptions(nameof(PostgresOptions)); var rabbitMqOptions = services.GetOptions(nameof(RabbitMqOptions)); var mongoOptions = services.GetOptions(nameof(MongoOptions)); - var logOptions = services.GetOptions(nameof(LogOptions)); var healthChecksBuilder = services.AddHealthChecks() .AddRabbitMQ( rabbitConnectionString: - $"amqp://{rabbitMqOptions.UserName}:{rabbitMqOptions.Password}@{rabbitMqOptions.HostName}") - .AddElasticsearch(logOptions.Elastic.ElasticServiceUrl); + $"amqp://{rabbitMqOptions.UserName}:{rabbitMqOptions.Password}@{rabbitMqOptions.HostName}"); if (mongoOptions.ConnectionString is not null) healthChecksBuilder.AddMongoDb(mongoOptions.ConnectionString); diff --git a/src/BuildingBlocks/Logging/ElasticOptions.cs b/src/BuildingBlocks/Logging/ElasticOptions.cs deleted file mode 100644 index 65e210ba..00000000 --- a/src/BuildingBlocks/Logging/ElasticOptions.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace BuildingBlocks.Logging; - -public class ElasticOptions -{ - public bool Enabled { get; set; } - public string ElasticServiceUrl { get; set; } - public string ElasticSearchIndex { get; set; } -} diff --git a/src/BuildingBlocks/Logging/Extensions.cs b/src/BuildingBlocks/Logging/Extensions.cs index ed1813eb..1c765c05 100644 --- a/src/BuildingBlocks/Logging/Extensions.cs +++ b/src/BuildingBlocks/Logging/Extensions.cs @@ -7,7 +7,6 @@ using Serilog; using Serilog.Events; using Serilog.Exceptions; -using Serilog.Sinks.Elasticsearch; using Serilog.Sinks.SpectreConsole; namespace BuildingBlocks.Logging @@ -18,10 +17,7 @@ public static WebApplicationBuilder AddCustomSerilog(this WebApplicationBuilder { builder.Host.UseSerilog((context, services, loggerConfiguration) => { - var environment = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT"); var logOptions = context.Configuration.GetSection(nameof(LogOptions)).Get(); - var appOptions = context.Configuration.GetSection(nameof(AppOptions)).Get(); - var logLevel = Enum.TryParse(logOptions.Level, true, out var level) ? level @@ -39,35 +35,6 @@ public static WebApplicationBuilder AddCustomSerilog(this WebApplicationBuilder .Enrich.FromLogContext() .ReadFrom.Configuration(context.Configuration); - if (logOptions.Elastic is { Enabled: true }) - { - loggerConfiguration.WriteTo.Elasticsearch( - new ElasticsearchSinkOptions(new Uri(logOptions.Elastic.ElasticServiceUrl)) - { - AutoRegisterTemplate = true, - IndexFormat = $"{appOptions.Name}-{environment?.ToLower(CultureInfo.CurrentCulture)}" - }); - } - - - if (logOptions?.Sentry is { Enabled: true }) - { - var minimumBreadcrumbLevel = Enum.TryParse(logOptions.Level, true, out var minBreadcrumbLevel) - ? minBreadcrumbLevel - : LogEventLevel.Information; - - var minimumEventLevel = Enum.TryParse(logOptions.Sentry.MinimumEventLevel, true, out var minEventLevel) - ? minEventLevel - : LogEventLevel.Error; - - loggerConfiguration.WriteTo.Sentry(o => - { - o.Dsn = logOptions.Sentry.Dsn; - o.MinimumBreadcrumbLevel = minimumBreadcrumbLevel; - o.MinimumEventLevel = minimumEventLevel; - }); - } - if (logOptions.File is { Enabled: true }) { var root = env.ContentRootPath; diff --git a/src/BuildingBlocks/Logging/LogOptions.cs b/src/BuildingBlocks/Logging/LogOptions.cs index 8fac194e..ee4f7775 100644 --- a/src/BuildingBlocks/Logging/LogOptions.cs +++ b/src/BuildingBlocks/Logging/LogOptions.cs @@ -3,9 +3,6 @@ namespace BuildingBlocks.Logging public class LogOptions { public string Level { get; set; } - public ElasticOptions Elastic { get; set; } - - public SentryOptions Sentry { get; set; } public FileOptions File { get; set; } public string LogTemplate { get; set; } } diff --git a/src/BuildingBlocks/Logging/SentryOptions.cs b/src/BuildingBlocks/Logging/SentryOptions.cs deleted file mode 100644 index 8556bbe2..00000000 --- a/src/BuildingBlocks/Logging/SentryOptions.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace BuildingBlocks.Logging; - -public class SentryOptions -{ - public bool Enabled { get; set; } - public string Dsn { get; set; } - public string MinimumBreadcrumbLevel { get; set; } - public string MinimumEventLevel { get; set; } -} diff --git a/src/BuildingBlocks/OpenTelemetry/Extensions.cs b/src/BuildingBlocks/OpenTelemetry/Extensions.cs deleted file mode 100644 index a31a3d96..00000000 --- a/src/BuildingBlocks/OpenTelemetry/Extensions.cs +++ /dev/null @@ -1,44 +0,0 @@ -using BuildingBlocks.Utils; -using BuildingBlocks.Web; -using Microsoft.Extensions.DependencyInjection; -using OpenTelemetry.Resources; -using OpenTelemetry.Trace; - -namespace BuildingBlocks.OpenTelemetry; - -using global::OpenTelemetry.Metrics; - -public static class Extensions -{ - public static IServiceCollection AddCustomOpenTelemetry(this IServiceCollection services) - { - services.AddOpenTelemetry() - .WithTracing(builder => builder - .AddGrpcClientInstrumentation() - .AddMassTransitInstrumentation() - .AddAspNetCoreInstrumentation() - .AddHttpClientInstrumentation() - .SetResourceBuilder(ResourceBuilder.CreateDefault() - .AddService(services.GetOptions("AppOptions").Name)) - .AddJaegerExporter()) - .WithMetrics(builder => - { - builder.AddPrometheusExporter(); - builder.AddMeter( - "Microsoft.AspNetCore.Hosting", - "Microsoft.AspNetCore.Server.Kestrel" - ); - builder.AddView("request-duration", - new ExplicitBucketHistogramConfiguration - { - Boundaries = new[] - { - 0, 0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10 - } - }); - } - ); - - return services; - } -} diff --git a/src/BuildingBlocks/OpenTelemetryCollector/ActivityExtensions.cs b/src/BuildingBlocks/OpenTelemetryCollector/ActivityExtensions.cs new file mode 100644 index 00000000..fb06a1c5 --- /dev/null +++ b/src/BuildingBlocks/OpenTelemetryCollector/ActivityExtensions.cs @@ -0,0 +1,175 @@ +using System.Diagnostics; +using System.Globalization; + +namespace BuildingBlocks.OpenTelemetryCollector; + +internal static class ActivityExtensions +{ + /// + /// Retrieves the tags from the parent of the current Activity, if available. + /// + /// The current Activity. + /// A dictionary containing the parent tags, or an empty dictionary if no parent tags are available. + public static Dictionary GetParentTags(this Activity activity) + { + ArgumentNullException.ThrowIfNull(activity); + + var parentTags = new Dictionary(); + + // Check if the current activity has a parent + var parentActivity = activity.Parent; + + if (parentActivity != null) + { + foreach (var tag in parentActivity.Tags) + { + parentTags[tag.Key] = tag.Value; + } + } + else + { + // If no parent Activity is available, check for links + foreach (var link in activity.Links) + { + // Extract tags from the first link's context (assuming it's the parent-like context) + if (link.Tags != null) + { + foreach (var tag in link.Tags) + { + parentTags[tag.Key] = tag.Value; + } + } + + // Break after processing the first link, as there should only be one parent context. + break; + } + } + + return parentTags; + } + + /// + /// Extracts important information from an Activity into an ActivityInfo object. + /// + /// The Activity from which to extract information. + /// An ActivityInfo object containing the extracted information. + public static ActivityInfo ExtractImportantInformation(this Activity activity) + { + ArgumentNullException.ThrowIfNull(activity); + + var activityInfo = new ActivityInfo + { + Name = activity.DisplayName, + StartTime = activity.StartTimeUtc, + Duration = activity.Duration, + Status = + activity.Tags.FirstOrDefault(tag => tag.Key == TelemetryTags.Tracing.Otel.StatusCode).Value + ?? "Unknown", + StatusDescription = activity + .Tags.FirstOrDefault(tag => tag.Key == TelemetryTags.Tracing.Otel.StatusDescription) + .Value, + Tags = activity.Tags.ToDictionary(tag => tag.Key, tag => tag.Value), + Events = activity + .Events.Select(e => new ActivityEventInfo + { + Name = e.Name, + Timestamp = e.Timestamp, + Attributes = e.Tags.ToDictionary(tag => tag.Key, tag => tag.Value), + }) + .ToList(), + TraceId = activity.TraceId.ToString(), + SpanId = activity.SpanId.ToString(), + }; + + return activityInfo; + } + + /// + /// Sets an "OK" status on the provided Activity, indicating a successful operation. + /// + /// The Activity to update. + /// An optional description of the successful operation. + /// The updated Activity with the status and tags set. + public static Activity SetOkStatus(this Activity activity, string? description = null) + { + ArgumentNullException.ThrowIfNull(activity); + + // Set the status of the activity to "OK" + activity.SetStatus(ActivityStatusCode.Ok, description); + + // Add telemetry tags for status + activity.SetTag( + TelemetryTags.Tracing.Otel.StatusCode, + nameof(ActivityStatusCode.Ok).ToUpper(CultureInfo.InvariantCulture) + ); + if (!string.IsNullOrEmpty(description)) + activity.SetTag(TelemetryTags.Tracing.Otel.StatusDescription, description); + + return activity; + } + + /// + /// Sets an "Unset" status on the provided Activity, indicating no explicit status was applied. + /// + /// The Activity to update. + /// An optional description of the unset status. + /// The updated Activity with the status and tags set. + public static Activity SetUnsetStatus(this Activity activity, string? description = null) + { + ArgumentNullException.ThrowIfNull(activity); + + // Set the status of the activity to "Unset" + activity.SetStatus(ActivityStatusCode.Unset, description); + + // Add telemetry tags for status + activity.SetTag( + TelemetryTags.Tracing.Otel.StatusCode, + nameof(ActivityStatusCode.Unset).ToUpper(CultureInfo.InvariantCulture) + ); + if (!string.IsNullOrEmpty(description)) + activity.SetTag(TelemetryTags.Tracing.Otel.StatusDescription, description); + + return activity; + } + + /// + /// Sets an "Error" status on the provided Activity, indicating a failed operation. + /// + /// The Activity to update. + /// The exception associated with the error, if available. + /// An optional description of the error. + /// The updated Activity with the status, error details, and tags set. + public static Activity SetErrorStatus(this Activity activity, System.Exception? exception, string? description = null) + { + ArgumentNullException.ThrowIfNull(activity); + + // Add telemetry tags for status + activity.SetTag( + TelemetryTags.Tracing.Otel.StatusCode, + nameof(ActivityStatusCode.Error).ToUpper(CultureInfo.InvariantCulture) + ); + if (!string.IsNullOrEmpty(description)) + activity.SetTag(TelemetryTags.Tracing.Otel.StatusDescription, description); + + // Add detailed exception tags, if an exception is provided + return activity.SetExceptionTags(exception); + } + + // See https://opentelemetry.io/docs/specs/otel/trace/semantic_conventions/exceptions/ + public static Activity SetExceptionTags(this Activity activity, System.Exception? ex) + { + if (ex is null) + { + return activity; + } + + activity.SetStatus(ActivityStatusCode.Error); + activity.AddException(ex); + + activity.AddTag(TelemetryTags.Tracing.Exception.Message, ex.Message); + activity.AddTag(TelemetryTags.Tracing.Exception.Stacktrace, ex.ToString()); + activity.AddTag(TelemetryTags.Tracing.Exception.Type, ex.GetType().FullName); + + return activity; + } +} diff --git a/src/BuildingBlocks/OpenTelemetryCollector/ActivityInfo.cs b/src/BuildingBlocks/OpenTelemetryCollector/ActivityInfo.cs new file mode 100644 index 00000000..0392f0f4 --- /dev/null +++ b/src/BuildingBlocks/OpenTelemetryCollector/ActivityInfo.cs @@ -0,0 +1,29 @@ +using System.Diagnostics; + +namespace BuildingBlocks.OpenTelemetryCollector; + +public class ActivityInfo +{ + public string Name { get; set; } = default!; + public DateTime StartTime { get; set; } + public TimeSpan Duration { get; set; } + public string Status { get; set; } = default!; + public string? StatusDescription { get; set; } + public IDictionary Tags { get; set; } = new Dictionary(); + public IList Events { get; set; } = new List(); + public string TraceId { get; set; } = default!; + public string SpanId { get; set; } = default!; + + public string? ParentId { get; set; } + + public ActivityContext? Parent { get; set; } + + public ActivityKind Kind { get; set; } +} + +public class ActivityEventInfo +{ + public string Name { get; set; } = default!; + public DateTimeOffset Timestamp { get; set; } + public IDictionary Attributes { get; set; } = new Dictionary(); +} diff --git a/src/BuildingBlocks/OpenTelemetryCollector/Behaviors/ObservabilityPipelineBehavior.cs b/src/BuildingBlocks/OpenTelemetryCollector/Behaviors/ObservabilityPipelineBehavior.cs new file mode 100644 index 00000000..aadb938f --- /dev/null +++ b/src/BuildingBlocks/OpenTelemetryCollector/Behaviors/ObservabilityPipelineBehavior.cs @@ -0,0 +1,85 @@ +using BuildingBlocks.Core.CQRS; +using BuildingBlocks.OpenTelemetryCollector.CoreDiagnostics.Commands; +using BuildingBlocks.OpenTelemetryCollector.CoreDiagnostics.Query; +using MediatR; + +namespace BuildingBlocks.OpenTelemetryCollector.Behaviors; + +public class ObservabilityPipelineBehavior( + CommandHandlerActivity commandActivity, + CommandHandlerMetrics commandMetrics, + QueryHandlerActivity queryActivity, + QueryHandlerMetrics queryMetrics +) : IPipelineBehavior + where TRequest : IRequest + where TResponse : notnull +{ + public async Task Handle(TRequest message, RequestHandlerDelegate next, CancellationToken cancellationToken) + { + var isCommand = message is IQuery; + var isQuery = message is ICommand; + + if (isCommand) + { + commandMetrics.StartExecuting(); + } + + if (isQuery) + { + queryMetrics.StartExecuting(); + } + + try + { + if (isCommand) + { + var commandResult = await commandActivity.Execute( + async (activity, ct) => + { + var response = await next(); + + return response; + }, + cancellationToken + ); + + commandMetrics.FinishExecuting(); + + return commandResult; + } + + if (isQuery) + { + var queryResult = await queryActivity.Execute( + async (activity, ct) => + { + var response = await next(); + + return response; + }, + cancellationToken + ); + + queryMetrics.FinishExecuting(); + + return queryResult; + } + } + catch (System.Exception) + { + if (isQuery) + { + queryMetrics.FailedCommand(); + } + + if (isCommand) + { + commandMetrics.FailedCommand(); + } + + throw; + } + + return await next(); + } +} diff --git a/src/BuildingBlocks/OpenTelemetryCollector/CoreDiagnostics/Commands/CommandHandlerActivity.cs b/src/BuildingBlocks/OpenTelemetryCollector/CoreDiagnostics/Commands/CommandHandlerActivity.cs new file mode 100644 index 00000000..c136ff34 --- /dev/null +++ b/src/BuildingBlocks/OpenTelemetryCollector/CoreDiagnostics/Commands/CommandHandlerActivity.cs @@ -0,0 +1,86 @@ +using System.Diagnostics; +using BuildingBlocks.Core.CQRS; +using BuildingBlocks.OpenTelemetryCollector.DiagnosticsProvider; + +namespace BuildingBlocks.OpenTelemetryCollector.CoreDiagnostics.Commands; + +public class CommandHandlerActivity(IDiagnosticsProvider diagnosticsProvider) +{ + public async Task Execute( + Func action, + CancellationToken cancellationToken + ) + { + var commandName = typeof(TCommand).Name; + var handlerType = typeof(TCommand) + .Assembly.GetTypes() + .FirstOrDefault(t => + t.GetInterfaces() + .Any(i => + i.IsGenericType + && i.GetGenericTypeDefinition() == typeof(ICommandHandler<,>) + && i.GetGenericArguments()[0] == typeof(TCommand) + ) + ); + var commandHandlerName = handlerType?.Name; + + // usually we use class/methodName + var activityName = $"{ObservabilityConstant.Components.CommandHandler}.{commandHandlerName}/{commandName}"; + + await diagnosticsProvider.ExecuteActivityAsync( + new CreateActivityInfo + { + Name = activityName, + ActivityKind = ActivityKind.Consumer, + Tags = new Dictionary + { + { TelemetryTags.Tracing.Application.Commands.Command, commandName }, + { TelemetryTags.Tracing.Application.Commands.CommandType, typeof(TCommand).FullName }, + { TelemetryTags.Tracing.Application.Commands.CommandHandler, commandHandlerName }, + { TelemetryTags.Tracing.Application.Commands.CommandHandlerType, handlerType?.FullName }, + }, + }, + action, + cancellationToken + ); + } + + public async Task Execute( + Func> action, + CancellationToken cancellationToken + ) + { + var commandName = typeof(TCommand).Name; + var handlerType = typeof(TCommand) + .Assembly.GetTypes() + .FirstOrDefault(t => + t.GetInterfaces() + .Any(i => + i.IsGenericType + && i.GetGenericTypeDefinition() == typeof(ICommandHandler<,>) + && i.GetGenericArguments()[0] == typeof(TCommand) + ) + ); + var commandHandlerName = handlerType?.Name; + + // usually we use class/methodName + var activityName = $"{ObservabilityConstant.Components.CommandHandler}.{commandHandlerName}/{commandName}"; + + return await diagnosticsProvider.ExecuteActivityAsync( + new CreateActivityInfo + { + Name = activityName, + ActivityKind = ActivityKind.Consumer, + Tags = new Dictionary + { + { TelemetryTags.Tracing.Application.Commands.Command, commandName }, + { TelemetryTags.Tracing.Application.Commands.CommandType, typeof(TCommand).FullName }, + { TelemetryTags.Tracing.Application.Commands.CommandHandler, commandHandlerName }, + { TelemetryTags.Tracing.Application.Commands.CommandHandlerType, handlerType?.FullName }, + }, + }, + action, + cancellationToken + ); + } +} diff --git a/src/BuildingBlocks/OpenTelemetryCollector/CoreDiagnostics/Commands/CommandHandlerMetrics.cs b/src/BuildingBlocks/OpenTelemetryCollector/CoreDiagnostics/Commands/CommandHandlerMetrics.cs new file mode 100644 index 00000000..1fc87acd --- /dev/null +++ b/src/BuildingBlocks/OpenTelemetryCollector/CoreDiagnostics/Commands/CommandHandlerMetrics.cs @@ -0,0 +1,157 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; +using BuildingBlocks.Core.CQRS; +using BuildingBlocks.OpenTelemetryCollector; +using BuildingBlocks.OpenTelemetryCollector.DiagnosticsProvider; + +namespace BuildingBlocks.OpenTelemetryCollector.CoreDiagnostics.Commands; + +public class CommandHandlerMetrics +{ + private readonly UpDownCounter _activeCommandsCounter; + private readonly Counter _totalCommandsNumber; + private readonly Counter _successCommandsNumber; + private readonly Counter _failedCommandsNumber; + private readonly Histogram _handlerDuration; + + private Stopwatch _timer; + + public CommandHandlerMetrics(IDiagnosticsProvider diagnosticsProvider) + { + _activeCommandsCounter = diagnosticsProvider.Meter.CreateUpDownCounter( + TelemetryTags.Metrics.Application.Commands.ActiveCount, + unit: "{active_commands}", + description: "Number of commands currently being handled" + ); + + _totalCommandsNumber = diagnosticsProvider.Meter.CreateCounter( + TelemetryTags.Metrics.Application.Commands.TotalExecutedCount, + unit: "{total_commands}", + description: "Total number of executed command that sent to command handlers" + ); + + _successCommandsNumber = diagnosticsProvider.Meter.CreateCounter( + TelemetryTags.Metrics.Application.Commands.SuccessCount, + unit: "{success_commands}", + description: "Number commands that handled successfully" + ); + + _failedCommandsNumber = diagnosticsProvider.Meter.CreateCounter( + TelemetryTags.Metrics.Application.Commands.FaildCount, + unit: "{failed_commands}", + description: "Number commands that handled with errors" + ); + + _handlerDuration = diagnosticsProvider.Meter.CreateHistogram( + TelemetryTags.Metrics.Application.Commands.HandlerDuration, + unit: "s", + description: "Measures the duration of command handler" + ); + } + + public void StartExecuting() + { + var commandName = typeof(TCommand).Name; + var handlerType = typeof(TCommand) + .Assembly.GetTypes() + .FirstOrDefault(t => + t.GetInterfaces() + .Any(i => + i.IsGenericType + && i.GetGenericTypeDefinition() == typeof(ICommandHandler<,>) + && i.GetGenericArguments()[0] == typeof(TCommand) + ) + ); + var commandHandlerName = handlerType?.Name; + + var tags = new TagList + { + { TelemetryTags.Tracing.Application.Commands.Command, commandName }, + { TelemetryTags.Tracing.Application.Commands.CommandType, typeof(TCommand).FullName }, + { TelemetryTags.Tracing.Application.Commands.CommandHandler, commandHandlerName }, + { TelemetryTags.Tracing.Application.Commands.CommandHandlerType, handlerType?.FullName }, + }; + + if (_activeCommandsCounter.Enabled) + { + _activeCommandsCounter.Add(1, tags); + } + + if (_totalCommandsNumber.Enabled) + { + _totalCommandsNumber.Add(1, tags); + } + + _timer = Stopwatch.StartNew(); + } + + public void FinishExecuting() + { + var commandName = typeof(TCommand).Name; + var handlerType = typeof(TCommand) + .Assembly.GetTypes() + .FirstOrDefault(t => + t.GetInterfaces() + .Any(i => + i.IsGenericType + && i.GetGenericTypeDefinition() == typeof(ICommandHandler<,>) + && i.GetGenericArguments()[0] == typeof(TCommand) + ) + ); + var commandHandlerName = handlerType?.Name; + + var tags = new TagList + { + { TelemetryTags.Tracing.Application.Commands.Command, commandName }, + { TelemetryTags.Tracing.Application.Commands.CommandType, typeof(TCommand).FullName }, + { TelemetryTags.Tracing.Application.Commands.CommandHandler, commandHandlerName }, + { TelemetryTags.Tracing.Application.Commands.CommandHandlerType, handlerType?.FullName }, + }; + + if (_activeCommandsCounter.Enabled) + { + _activeCommandsCounter.Add(-1, tags); + } + + if (!_handlerDuration.Enabled) + return; + + var elapsedTimeSeconds = _timer.Elapsed.Seconds; + + _handlerDuration.Record(elapsedTimeSeconds, tags); + + if (_successCommandsNumber.Enabled) + { + _successCommandsNumber.Add(1, tags); + } + } + + public void FailedCommand() + { + var commandName = typeof(TCommand).Name; + var handlerType = typeof(TCommand) + .Assembly.GetTypes() + .FirstOrDefault(t => + t.GetInterfaces() + .Any(i => + i.IsGenericType + && i.GetGenericTypeDefinition() == typeof(ICommandHandler<,>) + && i.GetGenericArguments()[0] == typeof(TCommand) + ) + ); + var commandHandlerName = handlerType?.Name; + + var tags = new TagList + { + { TelemetryTags.Tracing.Application.Commands.Command, commandName }, + { TelemetryTags.Tracing.Application.Commands.CommandType, typeof(TCommand).FullName }, + { TelemetryTags.Tracing.Application.Commands.CommandHandler, commandHandlerName }, + { TelemetryTags.Tracing.Application.Commands.CommandHandlerType, handlerType?.FullName }, + }; + + if (_failedCommandsNumber.Enabled) + { + _failedCommandsNumber.Add(1, tags); + } + } +} diff --git a/src/BuildingBlocks/OpenTelemetryCollector/CoreDiagnostics/Query/QueryHandlerActivity.cs b/src/BuildingBlocks/OpenTelemetryCollector/CoreDiagnostics/Query/QueryHandlerActivity.cs new file mode 100644 index 00000000..3d5b0372 --- /dev/null +++ b/src/BuildingBlocks/OpenTelemetryCollector/CoreDiagnostics/Query/QueryHandlerActivity.cs @@ -0,0 +1,47 @@ +using System.Diagnostics; +using BuildingBlocks.Core.CQRS; +using BuildingBlocks.OpenTelemetryCollector.DiagnosticsProvider; + +namespace BuildingBlocks.OpenTelemetryCollector.CoreDiagnostics.Query; + +public class QueryHandlerActivity(IDiagnosticsProvider diagnosticsProvider) +{ + public async Task Execute( + Func> action, + CancellationToken cancellationToken + ) + { + var queryName = typeof(TQuery).Name; + var handlerType = typeof(TQuery) + .Assembly.GetTypes() + .FirstOrDefault(t => + t.GetInterfaces() + .Any(i => + i.IsGenericType + && i.GetGenericTypeDefinition() == typeof(IQueryHandler<,>) + && i.GetGenericArguments()[0] == typeof(TQuery) + ) + ); + var queryHandlerName = handlerType?.Name; + + // usually we use class/methodName + var activityName = $"{ObservabilityConstant.Components.QueryHandler}.{queryHandlerName}/{queryName}"; + + return await diagnosticsProvider.ExecuteActivityAsync( + new CreateActivityInfo + { + Name = activityName, + ActivityKind = ActivityKind.Consumer, + Tags = new Dictionary + { + { TelemetryTags.Tracing.Application.Queries.Query, queryName }, + { TelemetryTags.Tracing.Application.Queries.QueryType, typeof(TQuery).FullName }, + { TelemetryTags.Tracing.Application.Queries.QueryHandler, queryHandlerName }, + { TelemetryTags.Tracing.Application.Queries.QueryHandlerType, handlerType?.FullName }, + }, + }, + action, + cancellationToken + ); + } +} diff --git a/src/BuildingBlocks/OpenTelemetryCollector/CoreDiagnostics/Query/QueryHandlerMetrics.cs b/src/BuildingBlocks/OpenTelemetryCollector/CoreDiagnostics/Query/QueryHandlerMetrics.cs new file mode 100644 index 00000000..eaea5dde --- /dev/null +++ b/src/BuildingBlocks/OpenTelemetryCollector/CoreDiagnostics/Query/QueryHandlerMetrics.cs @@ -0,0 +1,156 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; +using BuildingBlocks.Core.CQRS; +using BuildingBlocks.OpenTelemetryCollector.DiagnosticsProvider; + +namespace BuildingBlocks.OpenTelemetryCollector.CoreDiagnostics.Query; + +public class QueryHandlerMetrics +{ + private readonly UpDownCounter _activeQueriesCounter; + private readonly Counter _totalQueriesNumber; + private readonly Counter _successQueriesNumber; + private readonly Counter _failedQueriesNumber; + private readonly Histogram _handlerDuration; + + private Stopwatch _timer; + + public QueryHandlerMetrics(IDiagnosticsProvider diagnosticsProvider) + { + _activeQueriesCounter = diagnosticsProvider.Meter.CreateUpDownCounter( + TelemetryTags.Metrics.Application.Commands.ActiveCount, + unit: "{active_queries}", + description: "Number of queries currently being handled" + ); + + _totalQueriesNumber = diagnosticsProvider.Meter.CreateCounter( + TelemetryTags.Metrics.Application.Commands.TotalExecutedCount, + unit: "{total_queries}", + description: "Total number of executed query that sent to query handlers" + ); + + _successQueriesNumber = diagnosticsProvider.Meter.CreateCounter( + TelemetryTags.Metrics.Application.Commands.SuccessCount, + unit: "{success_queries}", + description: "Number queries that handled successfully" + ); + + _failedQueriesNumber = diagnosticsProvider.Meter.CreateCounter( + TelemetryTags.Metrics.Application.Commands.FaildCount, + unit: "{failed_queries}", + description: "Number queries that handled with errors" + ); + + _handlerDuration = diagnosticsProvider.Meter.CreateHistogram( + TelemetryTags.Metrics.Application.Commands.HandlerDuration, + unit: "s", + description: "Measures the duration of query handler" + ); + } + + public void StartExecuting() + { + var queryName = typeof(TQuery).Name; + var handlerType = typeof(TQuery) + .Assembly.GetTypes() + .FirstOrDefault(t => + t.GetInterfaces() + .Any(i => + i.IsGenericType + && i.GetGenericTypeDefinition() == typeof(IQueryHandler<,>) + && i.GetGenericArguments()[0] == typeof(TQuery) + ) + ); + var queryHandlerName = handlerType?.Name; + + var tags = new TagList + { + { TelemetryTags.Tracing.Application.Queries.Query, queryName }, + { TelemetryTags.Tracing.Application.Queries.QueryType, typeof(TQuery).FullName }, + { TelemetryTags.Tracing.Application.Queries.QueryHandler, queryHandlerName }, + { TelemetryTags.Tracing.Application.Queries.QueryHandlerType, handlerType?.FullName }, + }; + + if (_activeQueriesCounter.Enabled) + { + _activeQueriesCounter.Add(1, tags); + } + + if (_totalQueriesNumber.Enabled) + { + _totalQueriesNumber.Add(1, tags); + } + + _timer = Stopwatch.StartNew(); + } + + public void FinishExecuting() + { + var queryName = typeof(TQuery).Name; + var handlerType = typeof(TQuery) + .Assembly.GetTypes() + .FirstOrDefault(t => + t.GetInterfaces() + .Any(i => + i.IsGenericType + && i.GetGenericTypeDefinition() == typeof(IQueryHandler<,>) + && i.GetGenericArguments()[0] == typeof(TQuery) + ) + ); + var queryHandlerName = handlerType?.Name; + + var tags = new TagList + { + { TelemetryTags.Tracing.Application.Queries.Query, queryName }, + { TelemetryTags.Tracing.Application.Queries.QueryType, typeof(TQuery).FullName }, + { TelemetryTags.Tracing.Application.Queries.QueryHandler, queryHandlerName }, + { TelemetryTags.Tracing.Application.Queries.QueryHandlerType, handlerType?.FullName }, + }; + + if (_activeQueriesCounter.Enabled) + { + _activeQueriesCounter.Add(-1, tags); + } + + if (!_handlerDuration.Enabled) + return; + + var elapsedTimeSeconds = _timer.Elapsed.Seconds; + + _handlerDuration.Record(elapsedTimeSeconds, tags); + + if (_successQueriesNumber.Enabled) + { + _successQueriesNumber.Add(1, tags); + } + } + + public void FailedCommand() + { + var queryName = typeof(TQuery).Name; + var handlerType = typeof(TQuery) + .Assembly.GetTypes() + .FirstOrDefault(t => + t.GetInterfaces() + .Any(i => + i.IsGenericType + && i.GetGenericTypeDefinition() == typeof(IQueryHandler<,>) + && i.GetGenericArguments()[0] == typeof(TQuery) + ) + ); + var queryHandlerName = handlerType?.Name; + + var tags = new TagList + { + { TelemetryTags.Tracing.Application.Queries.Query, queryName }, + { TelemetryTags.Tracing.Application.Queries.QueryType, typeof(TQuery).FullName }, + { TelemetryTags.Tracing.Application.Queries.QueryHandler, queryHandlerName }, + { TelemetryTags.Tracing.Application.Queries.QueryHandlerType, handlerType?.FullName }, + }; + + if (_failedQueriesNumber.Enabled) + { + _failedQueriesNumber.Add(1, tags); + } + } +} diff --git a/src/BuildingBlocks/OpenTelemetryCollector/CreateActivityInfo.cs b/src/BuildingBlocks/OpenTelemetryCollector/CreateActivityInfo.cs new file mode 100644 index 00000000..bbaa8ab8 --- /dev/null +++ b/src/BuildingBlocks/OpenTelemetryCollector/CreateActivityInfo.cs @@ -0,0 +1,12 @@ +using System.Diagnostics; + +namespace BuildingBlocks.OpenTelemetryCollector; + +public class CreateActivityInfo +{ + public required string Name { get; set; } + public IDictionary Tags { get; set; } = new Dictionary(); + public string? ParentId { get; set; } + public ActivityContext? Parent { get; set; } + public required ActivityKind ActivityKind = ActivityKind.Internal; +} diff --git a/src/BuildingBlocks/OpenTelemetryCollector/DiagnosticsProvider/CustomeDiagnosticsProvider.cs b/src/BuildingBlocks/OpenTelemetryCollector/DiagnosticsProvider/CustomeDiagnosticsProvider.cs new file mode 100644 index 00000000..abbcc7f3 --- /dev/null +++ b/src/BuildingBlocks/OpenTelemetryCollector/DiagnosticsProvider/CustomeDiagnosticsProvider.cs @@ -0,0 +1,132 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; +using System.Reflection; +using Microsoft.Extensions.Options; + +namespace BuildingBlocks.OpenTelemetryCollector.DiagnosticsProvider; + +public class CustomeDiagnosticsProvider(IMeterFactory meterFactory, IOptions options) + : IDiagnosticsProvider +{ + private readonly Version? _version = Assembly.GetCallingAssembly().GetName().Version; + private ActivitySource? _activitySource; + private ActivityListener? _listener; + private Meter? _meter; + + public string InstrumentationName { get; } = options.Value.InstrumentationName ?? throw new ArgumentException("InstrumentationName cannot be null or empty."); + + // https://learn.microsoft.com/en-us/dotnet/core/diagnostics/distributed-tracing-instrumentation-walkthroughs + public ActivitySource ActivitySource + { + get + { + if (_activitySource != null) + return _activitySource; + + _activitySource = new(InstrumentationName, _version?.ToString()); + + _listener = new ActivityListener + { + ShouldListenTo = x => true, + Sample = (ref ActivityCreationOptions _) => ActivitySamplingResult.AllDataAndRecorded, + }; + ActivitySource.AddActivityListener(_listener); + + return _activitySource; + } + } + + // https://learn.microsoft.com/en-us/dotnet/core/diagnostics/metrics-instrumentation + public Meter Meter + { + get + { + if (_meter != null) + return _meter; + + _meter = meterFactory.Create(InstrumentationName, _version?.ToString()); + + return _meter; + } + } + + public async Task ExecuteActivityAsync( + CreateActivityInfo createActivityInfo, + Func action, + CancellationToken cancellationToken = default + ) + { + if (!options.Value.TracingEnabled) + { + await action(null, cancellationToken); + + return; + } + + using var activity = + ActivitySource + .CreateActivity( + name: $"{InstrumentationName}.{createActivityInfo.Name}", + kind: createActivityInfo.ActivityKind, + parentContext: createActivityInfo.Parent ?? default, + idFormat: ActivityIdFormat.W3C, + tags: createActivityInfo.Tags + ) + ?.Start() ?? Activity.Current; + + try + { + await action(activity!, cancellationToken); + activity?.SetOkStatus(); + } + catch (System.Exception ex) + { + activity?.SetErrorStatus(ex); + throw; + } + } + + public async Task ExecuteActivityAsync( + CreateActivityInfo createActivityInfo, + Func> action, + CancellationToken cancellationToken = default + ) + { + if (!options.Value.TracingEnabled) + { + return await action(null, cancellationToken); + } + + using var activity = + ActivitySource + .CreateActivity( + name: $"{InstrumentationName}.{createActivityInfo.Name}", + kind: createActivityInfo.ActivityKind, + parentContext: createActivityInfo.Parent ?? default, + idFormat: ActivityIdFormat.W3C, + tags: createActivityInfo.Tags + ) + ?.Start() ?? Activity.Current; + + try + { + var result = await action(activity!, cancellationToken); + + activity?.SetOkStatus(); + + return result; + } + catch (System.Exception ex) + { + activity?.SetErrorStatus(ex); + throw; + } + } + + public void Dispose() + { + _listener?.Dispose(); + _meter?.Dispose(); + _activitySource?.Dispose(); + } +} diff --git a/src/BuildingBlocks/OpenTelemetryCollector/DiagnosticsProvider/IDiagnosticsProvider.cs b/src/BuildingBlocks/OpenTelemetryCollector/DiagnosticsProvider/IDiagnosticsProvider.cs new file mode 100644 index 00000000..32cb70dd --- /dev/null +++ b/src/BuildingBlocks/OpenTelemetryCollector/DiagnosticsProvider/IDiagnosticsProvider.cs @@ -0,0 +1,23 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; + +namespace BuildingBlocks.OpenTelemetryCollector.DiagnosticsProvider; + +public interface IDiagnosticsProvider : IDisposable +{ + string InstrumentationName { get; } + ActivitySource ActivitySource { get; } + Meter Meter { get; } + + Task ExecuteActivityAsync( + CreateActivityInfo createActivityInfo, + Func action, + CancellationToken cancellationToken = default + ); + + Task ExecuteActivityAsync( + CreateActivityInfo createActivityInfo, + Func> action, + CancellationToken cancellationToken = default + ); +} diff --git a/src/BuildingBlocks/OpenTelemetryCollector/Extensions.cs b/src/BuildingBlocks/OpenTelemetryCollector/Extensions.cs new file mode 100644 index 00000000..04bf73ba --- /dev/null +++ b/src/BuildingBlocks/OpenTelemetryCollector/Extensions.cs @@ -0,0 +1,358 @@ +using System.Diagnostics; +using System.Reflection; +using BuildingBlocks.OpenTelemetryCollector.CoreDiagnostics.Commands; +using BuildingBlocks.OpenTelemetryCollector.CoreDiagnostics.Query; +using BuildingBlocks.OpenTelemetryCollector.DiagnosticsProvider; +using BuildingBlocks.Web; +using Grafana.OpenTelemetry; +using MassTransit.Logging; +using MassTransit.Monitoring; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http.Features; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Npgsql; +using OpenTelemetry; +using OpenTelemetry.Exporter; +using OpenTelemetry.Logs; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; + +namespace BuildingBlocks.OpenTelemetryCollector; + +// https://learn.microsoft.com/en-us/dotnet/core/diagnostics/observability-otlp-example +// https://learn.microsoft.com/en-us/dotnet/core/diagnostics/observability-prgrja-example +// https://learn.microsoft.com/en-us/dotnet/core/diagnostics/observability-prgrja-example +// https://blog.codingmilitia.com/2023/09/05/observing-dotnet-microservices-with-opentelemetry-logs-traces-metrics/ +public static class Extensions +{ + public static WebApplicationBuilder AddCustomObservability(this WebApplicationBuilder builder) + { + + Activity.DefaultIdFormat = ActivityIdFormat.W3C; + builder.Services.AddSingleton(); + builder.AddCoreDiagnostics(); + + builder.Services.AddValidateOptions(); + var observabilityOptions = builder.Services.GetOptions(nameof(ObservabilityOptions)); + + // InstrumentationName property option is mandatory and can't be empty + ArgumentException.ThrowIfNullOrEmpty(observabilityOptions.InstrumentationName); + ObservabilityConstant.InstrumentationName = observabilityOptions.InstrumentationName; + + if (observabilityOptions is { MetricsEnabled: false, TracingEnabled: false, LoggingEnabled: false }) + { + return builder; + } + + void ConfigureResourceBuilder(ResourceBuilder resourceBuilder) + { + resourceBuilder.AddAttributes([new("service.environment", builder.Environment.EnvironmentName)]); + + resourceBuilder.AddService( + serviceName: observabilityOptions.ServiceName ?? builder.Environment.ApplicationName, + serviceVersion: Assembly.GetCallingAssembly().GetName().Version?.ToString() ?? "unknown", + serviceInstanceId: Environment.MachineName + ); + } + + if (observabilityOptions.LoggingEnabled) + { + // logging + // opentelemtry logging works with .net default logging providers and doesn't work for `serilog`, in serilog we should enable `WriteToProviders=true` + builder.Logging.AddOpenTelemetry(options => + { + var resourceBuilder = ResourceBuilder.CreateDefault(); + ConfigureResourceBuilder(resourceBuilder); + options.SetResourceBuilder(resourceBuilder); + + options.IncludeFormattedMessage = true; + options.IncludeScopes = true; + // this allows the state value passed to the logger.Log method to be parsed, in case it isn't a collection of KeyValuePair, which is the case when we use things like logger.LogInformation. + options.ParseStateValues = true; + // which means the message wouldn't have the placeholders replaced + options.IncludeFormattedMessage = true; + + // add some metadata to exported logs + options.SetResourceBuilder( + ResourceBuilder + .CreateDefault() + .AddService( + observabilityOptions.ServiceName ?? builder.Environment.ApplicationName, + serviceVersion: Assembly.GetCallingAssembly().GetName().Version?.ToString() ?? "unknown", + serviceInstanceId: Environment.MachineName + ) + ); + + options.AddLoggingExporters(observabilityOptions); + }); + } + + if (observabilityOptions is { MetricsEnabled: false, TracingEnabled: false }) + { + return builder; + } + + OpenTelemetryBuilder otel = null!; + + if (observabilityOptions.MetricsEnabled || observabilityOptions.TracingEnabled) + { + // metrics and tracing + otel = builder.Services.AddOpenTelemetry(); + otel.ConfigureResource(ConfigureResourceBuilder); + } + + if (observabilityOptions.MetricsEnabled) + { + otel.WithMetrics(metrics => + { + metrics + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddProcessInstrumentation() + .AddRuntimeInstrumentation() + .AddMeter(InstrumentationOptions.MeterName) + .AddMeter(observabilityOptions.InstrumentationName) + // metrics provides by ASP.NET Core in .NET 8 + .AddView( + "http.server.request.duration", + new ExplicitBucketHistogramConfiguration + { + Boundaries = [0, 0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10], + } + ) + .AddMeter("System.Runtime") + .AddMeter("Microsoft.AspNetCore.Hosting") + .AddMeter("Microsoft.AspNetCore.Server.Kestrel"); + + AddMetricsExporter(observabilityOptions, metrics); + }); + } + + if (observabilityOptions.TracingEnabled) + { + otel.WithTracing(tracing => + { + if (builder.Environment.IsDevelopment()) + { + // We want to view all traces in development + tracing.SetSampler(new AlwaysOnSampler()); + } + + tracing + .SetErrorStatusOnException() + .AddAspNetCoreInstrumentation(options => + { + options.RecordException = true; + }) + .AddGrpcClientInstrumentation() + .AddHttpClientInstrumentation(instrumentationOptions => + { + instrumentationOptions.RecordException = true; + }) + .AddEntityFrameworkCoreInstrumentation(instrumentationOptions => + { + instrumentationOptions.SetDbStatementForText = true; + }) + .AddSource(DiagnosticHeaders.DefaultListenerName) + .AddNpgsql() + // `AddSource` for adding custom activity sources + .AddSource(observabilityOptions.InstrumentationName) + // metrics provides by ASP.NET Core in .NET 8 + .AddSource("Microsoft.AspNetCore.Hosting") + .AddSource("Microsoft.AspNetCore.Server.Kestrel"); + + AddTracingExporter(observabilityOptions, tracing); + }); + } + + return builder; + } + + + public static WebApplication UseCustomObservability(this WebApplication app) + { + var options = app.Services.GetRequiredService>().Value; + + app.Use( + async (context, next) => + { + var metricsFeature = context.Features.Get(); + if (metricsFeature != null && context.Request.Path is { Value: "/metrics" or "/health" }) + { + metricsFeature.MetricsDisabled = true; + } + + await next(context); + } + ); + + if (options.UsePrometheusExporter) + { + // export application metrics in `/metrics` endpoint and should scrape in the Prometheus config file and `scrape_configs` + // https://github.com/open-telemetry/opentelemetry-dotnet/tree/e330e57b04fa3e51fe5d63b52bfff891fb5b7961/src/OpenTelemetry.Exporter.Prometheus.AspNetCore + app.UseOpenTelemetryPrometheusScrapingEndpoint(); // http://localhost:4000/metrics + } + + return app; + } + + private static void AddTracingExporter(ObservabilityOptions observabilityOptions, TracerProviderBuilder tracing) + { + if (observabilityOptions.UseJaegerExporter) + { + ArgumentNullException.ThrowIfNull(observabilityOptions.JaegerOptions); + // https://github.com/open-telemetry/opentelemetry-dotnet/tree/e330e57b04fa3e51fe5d63b52bfff891fb5b7961/docs/trace/getting-started-jaeger + // `OpenTelemetry.Exporter.Jaeger` package and `AddJaegerExporter` to use Http endpoint (http://localhost:14268/api/traces) is deprecated, and we should use `OpenTelemetry.Exporter.OpenTelemetryProtocol` and `AddOtlpExporter` with OTLP port `4317` on Jaeger + // tracing.AddJaegerExporter( + // x => x.Endpoint = new Uri(observabilityOptions.JaegerOptions.HttpExporterEndpoint)); // http://localhost:14268/api/traces + tracing.AddOtlpExporter(x => + { + x.Endpoint = new Uri(observabilityOptions.JaegerOptions.OTLPGrpcExporterEndpoint); + x.Protocol = OtlpExportProtocol.Grpc; + }); + } + + if (observabilityOptions.UseZipkinExporter) + { + ArgumentNullException.ThrowIfNull(observabilityOptions.ZipkinOptions); + // https://github.com/open-telemetry/opentelemetry-dotnet/tree/e330e57b04fa3e51fe5d63b52bfff891fb5b7961/src/OpenTelemetry.Exporter.Zipkin + tracing.AddZipkinExporter(x => + x.Endpoint = new Uri(observabilityOptions.ZipkinOptions.HttpExporterEndpoint) + ); // "http://localhost:9411/api/v2/spans" + } + + if (observabilityOptions.UseConsoleExporter) + { + tracing.AddConsoleExporter(); + } + + if (observabilityOptions.UseOTLPExporter) + { + ArgumentNullException.ThrowIfNull(observabilityOptions.OTLPOptions); + tracing.AddOtlpExporter(x => + { + x.Endpoint = new Uri(observabilityOptions.OTLPOptions.OTLPGrpcExporterEndpoint); + x.Protocol = OtlpExportProtocol.Grpc; + }); + } + + if (observabilityOptions.UseAspireOTLPExporter) + { + // we can just one `AddOtlpExporter` and in development use `aspire-dashboard` OTLP endpoint address as `OTLPExporterEndpoint` and in production we can use `otel-collector` OTLP endpoint address + ArgumentNullException.ThrowIfNull(observabilityOptions.OTLPOptions); + tracing.AddOtlpExporter(x => + { + x.Endpoint = new Uri(observabilityOptions.AspireDashboardOTLPOptions.OTLPGrpcExporterEndpoint); + x.Protocol = OtlpExportProtocol.Grpc; + }); + } + + if (observabilityOptions.UseGrafanaExporter) + { + // https://github.com/grafana/grafana-opentelemetry-dotnet/blob/main/docs/configuration.md#aspnet-core + // https://github.com/grafana/grafana-opentelemetry-dotnet/ + // https://github.com/grafana/grafana-opentelemetry-dotnet/blob/main/docs/configuration.md#sending-to-an-agent-or-collector-via-otlp + // https://grafana.com/docs/grafana-cloud/monitor-applications/application-observability/instrument/dotnet/ + tracing.UseGrafana(); + } + } + + private static void AddMetricsExporter(ObservabilityOptions observabilityOptions, MeterProviderBuilder metrics) + { + if (observabilityOptions.UsePrometheusExporter) + { + // https://github.com/open-telemetry/opentelemetry-dotnet/tree/e330e57b04fa3e51fe5d63b52bfff891fb5b7961/src/OpenTelemetry.Exporter.Prometheus.AspNetCore + // for exporting app metrics to `/metrics` endpoint + metrics.AddPrometheusExporter(o => o.DisableTotalNameSuffixForCounters = true); // http://localhost:4000/metrics + } + + if (observabilityOptions.UseConsoleExporter) + { + metrics.AddConsoleExporter(); + } + + if (observabilityOptions.UseOTLPExporter) + { + ArgumentNullException.ThrowIfNull(observabilityOptions.OTLPOptions); + metrics.AddOtlpExporter(x => + { + x.Endpoint = new Uri(observabilityOptions.OTLPOptions.OTLPGrpcExporterEndpoint); + x.Protocol = OtlpExportProtocol.Grpc; + }); + } + + if (observabilityOptions.UseAspireOTLPExporter) + { + // we can just one `AddOtlpExporter` and in development use `aspire-dashboard` OTLP endpoint address as `OTLPExporterEndpoint` and in production we can use `otel-collector` OTLP endpoint address + ArgumentNullException.ThrowIfNull(observabilityOptions.OTLPOptions); + metrics.AddOtlpExporter(x => + { + x.Endpoint = new Uri(observabilityOptions.AspireDashboardOTLPOptions.OTLPGrpcExporterEndpoint); + x.Protocol = OtlpExportProtocol.Grpc; + }); + } + + if (observabilityOptions.UseGrafanaExporter) + { + // https://github.com/grafana/grafana-opentelemetry-dotnet/blob/main/docs/configuration.md#aspnet-core + // https://github.com/grafana/grafana-opentelemetry-dotnet/ + // https://github.com/grafana/grafana-opentelemetry-dotnet/blob/main/docs/configuration.md#sending-to-an-agent-or-collector-via-otlp + // https://grafana.com/docs/grafana-cloud/monitor-applications/application-observability/instrument/dotnet/ + metrics.UseGrafana(); + } + } + + private static void AddLoggingExporters( + this OpenTelemetryLoggerOptions openTelemetryLoggerOptions, + ObservabilityOptions observabilityOptions + ) + { + if (observabilityOptions.UseOTLPExporter) + { + ArgumentNullException.ThrowIfNull(observabilityOptions.OTLPOptions); + openTelemetryLoggerOptions.AddOtlpExporter(options => + { + options.Endpoint = new Uri(observabilityOptions.OTLPOptions.OTLPGrpcExporterEndpoint); + options.Protocol = OtlpExportProtocol.Grpc; + }); + } + + if (observabilityOptions.UseAspireOTLPExporter) + { + // we can just one `AddOtlpExporter` and in development use `aspire-dashboard` OTLP endpoint address as `OTLPExporterEndpoint` and in production we can use `otel-collector` OTLP endpoint address + ArgumentNullException.ThrowIfNull(observabilityOptions.OTLPOptions); + openTelemetryLoggerOptions.AddOtlpExporter(x => + { + x.Endpoint = new Uri(observabilityOptions.AspireDashboardOTLPOptions.OTLPGrpcExporterEndpoint); + x.Protocol = OtlpExportProtocol.Grpc; + }); + } + + if (observabilityOptions.UseGrafanaExporter) + { + // https://github.com/grafana/grafana-opentelemetry-dotnet/ + // https://github.com/grafana/grafana-opentelemetry-dotnet/blob/main/docs/configuration.md#aspnet-core + // https://grafana.com/docs/grafana-cloud/monitor-applications/application-observability/instrument/dotnet/ + openTelemetryLoggerOptions.UseGrafana(); + } + + if (observabilityOptions.UseConsoleExporter) + { + openTelemetryLoggerOptions.AddConsoleExporter(); + } + } + + private static WebApplicationBuilder AddCoreDiagnostics(this WebApplicationBuilder builder) + { + builder.Services.AddTransient(); + builder.Services.AddTransient(); + builder.Services.AddTransient(); + builder.Services.AddTransient(); + + return builder; + } +} diff --git a/src/BuildingBlocks/OpenTelemetryCollector/ObservabilityConstant.cs b/src/BuildingBlocks/OpenTelemetryCollector/ObservabilityConstant.cs new file mode 100644 index 00000000..319a97c9 --- /dev/null +++ b/src/BuildingBlocks/OpenTelemetryCollector/ObservabilityConstant.cs @@ -0,0 +1,16 @@ +namespace BuildingBlocks.OpenTelemetryCollector; + +public static class ObservabilityConstant +{ + public static string InstrumentationName = default!; + + public static class Components + { + public const string CommandHandler = "CommandHandler"; + public const string QueryHandler = "QueryHandler"; + public const string EventStore = "EventStore"; + public const string Producer = "Producer"; + public const string Consumer = "Consumer"; + public const string EventHandler = "EventHandler"; + } +} diff --git a/src/BuildingBlocks/OpenTelemetryCollector/ObservabilityOptions.cs b/src/BuildingBlocks/OpenTelemetryCollector/ObservabilityOptions.cs new file mode 100644 index 00000000..dca919a2 --- /dev/null +++ b/src/BuildingBlocks/OpenTelemetryCollector/ObservabilityOptions.cs @@ -0,0 +1,47 @@ +namespace BuildingBlocks.OpenTelemetryCollector; + +public class ObservabilityOptions +{ + public string InstrumentationName { get; set; } = default!; + public string? ServiceName { get; set; } + public bool MetricsEnabled { get; set; } = true; + public bool TracingEnabled { get; set; } = true; + public bool LoggingEnabled { get; set; } = true; + public bool UsePrometheusExporter { get; set; } = true; + public bool UseOTLPExporter { get; set; } = true; + public bool UseAspireOTLPExporter { get; set; } = true; + public bool UseGrafanaExporter { get; set; } + public bool UseConsoleExporter { get; set; } + public bool UseJaegerExporter { get; set; } + public bool UseZipkinExporter { get; set; } + public ZipkinOptions ZipkinOptions { get; set; } = default!; + public JaegerOptions JaegerOptions { get; set; } = default!; + public OTLPOptions OTLPOptions { get; set; } = default!; + public AspireDashboardOTLPOptions AspireDashboardOTLPOptions { get; set; } = default!; +} + +// https://github.com/open-telemetry/opentelemetry-dotnet/blob/main/src/OpenTelemetry.Exporter.Zipkin/README.md +public class ZipkinOptions +{ + /// + /// Gets or sets endpoint address to receive telemetry + /// + public string HttpExporterEndpoint { get; set; } = "http://localhost:9411/api/v2/spans"; +} + +public class JaegerOptions +{ + public string OTLPGrpcExporterEndpoint { get; set; } = "http://localhost:14317"; + public string HttpExporterEndpoint { get; set; } = "http://localhost:14268/api/traces"; +} + +public class OTLPOptions +{ + public string OTLPGrpcExporterEndpoint { get; set; } = "http://localhost:4317"; + public string OTLPHttpExporterEndpoint { get; set; } = "http://localhost:4318"; +} + +public class AspireDashboardOTLPOptions +{ + public string OTLPGrpcExporterEndpoint { get; set; } = "http://localhost:4319"; +} diff --git a/src/BuildingBlocks/OpenTelemetryCollector/TelemetryTags.cs b/src/BuildingBlocks/OpenTelemetryCollector/TelemetryTags.cs new file mode 100644 index 00000000..7cf27874 --- /dev/null +++ b/src/BuildingBlocks/OpenTelemetryCollector/TelemetryTags.cs @@ -0,0 +1,265 @@ +namespace BuildingBlocks.OpenTelemetryCollector; + +/// +/// Telemetry tags use for adding tags to activities as tag name +/// +public static class TelemetryTags +{ + // https://opentelemetry.io/docs/specs/semconv/general/trace/ + // https://opentelemetry.io/docs/specs/semconv/general/attribute-naming/ + public static class Tracing + { + // https://opentelemetry.io/docs/specs/semconv/resource/#service + // https://opentelemetry.io/docs/specs/semconv/attributes-registry/peer/#peer-attributes + public static class Service + { + public const string PeerService = "peer.service"; + public const string Name = "service.name"; + public const string InstanceId = "service.instance.id"; + public const string Version = "service.version"; + public const string NameSpace = "service.namespace"; + } + + // https://opentelemetry.io/docs/specs/semconv/attributes-registry/messaging/#general-messaging-attributes + // https://opentelemetry.io/docs/specs/semconv/messaging/messaging-spans/ + public static class Messaging + { + // https://opentelemetry.io/docs/specs/semconv/attributes-registry/messaging/#messaging-operation-type + public static class OperationType + { + public const string Key = "messaging.operation.type"; + public const string Receive = "receive"; + public const string Send = "send"; + public const string Process = "process"; + } + + // https://opentelemetry.io/docs/specs/semconv/attributes-registry/messaging/#messaging-system + public static class System + { + public const string Key = "messaging.system"; + public const string ActiveMQ = "activemq"; + public const string RabbitMQ = "rabbitmq"; + public const string AwsSqs = "aws_sqs"; + public const string EventGrid = "eventgrid"; + public const string EventHubs = "eventhubs"; + public const string GcpPubSub = "gcp_pubsub"; + public const string Kafka = "kafka"; + public const string Pulsar = "pulsar"; + public const string ServiceBus = "servicebus"; + } + + public const string Destination = "messaging.destination"; + public const string DestinationKind = "messaging.destination_kind"; + public const string Url = "messaging.url"; + public const string MessageId = "messaging.message_id"; + public const string ConversationId = "messaging.conversation_id"; + public const string CorrelationId = "messaging.correlation_id"; + public const string CausationId = "messaging.causation_id"; + public const string Operation = "messaging.operation"; + public const string OperationName = "messaging.operation.name"; + public const string DestinationName = "messaging.destination.name"; + public const string ConsumerGroup = "messaging.consumer.group.name"; + public const string DestinationPartition = "messaging.destination.partition.id"; + + // https://opentelemetry.io/docs/specs/semconv/attributes-registry/messaging/#rabbitmq-attributes + // https://opentelemetry.io/docs/specs/semconv/messaging/rabbitmq/ + public static class RabbitMQ + { + public const string RoutingKey = "messaging.rabbitmq.destination.routing_key"; + public const string DeliveryTag = "messaging.rabbitmq.message.delivery_tag"; + + public static IDictionary ProducerTags( + string serviceName, + string topicName, + string routingKey, + string? deliveryTag = null + ) => + new Dictionary + { + { System.Key, System.Kafka }, + { DeliveryTag, deliveryTag }, + { Destination, topicName }, + { OperationType.Key, OperationType.Send }, + { Service.Name, serviceName }, + { RoutingKey, routingKey }, + }; + + public static IDictionary ConsumerTags( + string serviceName, + string topicName, + string routingKey, + string? consumerGroup = null + ) => + new Dictionary + { + { System.Key, System.Kafka }, + { Destination, topicName }, + { OperationType.Key, OperationType.Receive }, + { Service.Name, serviceName }, + { ConsumerGroup, consumerGroup }, + { RoutingKey, routingKey }, + }; + } + + // https://opentelemetry.io/docs/specs/semconv/attributes-registry/messaging/#kafka-attributes + // https://opentelemetry.io/docs/specs/semconv/messaging/kafka/ + public static class Kafka + { + public const string MessageKey = "messaging.kafka.message.key"; + public const string Tombstone = "messaging.kafka.message.tombstone"; + public const string Offset = "messaging.kafka.offset"; + + public static IDictionary ProducerTags( + string serviceName, + string topicName, + string messageKey + ) => + new Dictionary + { + { System.Key, System.Kafka }, + { Destination, topicName }, + { OperationType.Key, OperationType.Send }, + { Service.Name, serviceName }, + { MessageKey, messageKey }, + }; + + public static IDictionary ConsumerTags( + string serviceName, + string topicName, + string messageKey, + string partitionName, + string consumerGroup + ) => + new Dictionary + { + { System.Key, System.Kafka }, + { Destination, topicName }, + { OperationType.Key, OperationType.Receive }, + { Service.Name, serviceName }, + { MessageKey, messageKey }, + { DestinationPartition, partitionName }, + { ConsumerGroup, consumerGroup }, + }; + } + } + + // https://opentelemetry.io/docs/specs/semconv/database/database-spans/#common-attributes + // https://opentelemetry.io/docs/specs/semconv/database/postgresql/#attributes + public static class Db + { + public const string System = "db.system"; + public const string ConnectionString = "db.connection_string"; + public const string User = "db.user"; + public const string MsSqlInstanceName = "db.mssql.instance_name"; + public const string Name = "db.name"; + public const string Statement = "db.statement"; + public const string Operation = "db.operation"; + public const string Instance = "db.instance"; + public const string Url = "db.url"; + public const string CassandraKeyspace = "db.cassandra.keyspace"; + public const string RedisDatabaseIndex = "db.redis.database_index"; + public const string MongoDbCollection = "db.mongodb.collection"; + } + + // https://opentelemetry.io/docs/specs/semconv/exceptions/exceptions-spans/#exception-event + public static class Exception + { + public const string EventName = "exception"; + public const string Type = "exception.type"; + public const string Message = "exception.message"; + public const string Stacktrace = "exception.stacktrace"; + } + + // https://opentelemetry.io/docs/specs/semconv/attributes-registry/otel/#otel-attributes + public static class Otel + { + public const string StatusCode = "otel.status_code"; + public const string StatusDescription = "otel.status_description"; + } + + public static class Message + { + public const string Type = "message.type"; + public const string Id = "message.id"; + } + + public static class Application + { + public static string AppService = $"{ObservabilityConstant.InstrumentationName}.appservice"; + public static string Consumer = $"{ObservabilityConstant.InstrumentationName}.consumer"; + public static string Producer = $"{ObservabilityConstant.InstrumentationName}.producer"; + + public static class Commands + { + public static string Command = $"{ObservabilityConstant.InstrumentationName}.command"; + public static string CommandType = $"{Command}.type"; + public static string CommandHandler = $"{Command}.handler"; + public static string CommandHandlerType = $"{CommandHandler}.type"; + } + + public static class Queries + { + public static string Query = $"{ObservabilityConstant.InstrumentationName}.query"; + public static string QueryType = $"{Query}.type"; + public static string QueryHandler = $"{Query}.handler"; + public static string QueryHandlerType = $"{QueryHandler}.type"; + } + + public static class Events + { + public static string Event = $"{ObservabilityConstant.InstrumentationName}.event"; + public static string EventType = $"{Event}.type"; + public static string EventHandler = $"{Event}.handler"; + public static string EventHandlerType = $"{EventHandler}.type"; + } + } + } + + // https://opentelemetry.io/docs/specs/semconv/general/metrics/ + // https://opentelemetry.io/docs/specs/semconv/general/attribute-naming/ + public static class Metrics + { + public static class Application + { + public static string AppService = $"{ObservabilityConstant.InstrumentationName}.appservice"; + public static string Consumer = $"{ObservabilityConstant.InstrumentationName}.consumer"; + public static string Producer = $"{ObservabilityConstant.InstrumentationName}.producer"; + + public static class Commands + { + public static string Command = $"{ObservabilityConstant.InstrumentationName}.command"; + public static string CommandType = $"{Command}.type"; + public static string CommandHandler = $"{Command}.handler"; + public static string SuccessCount = $"{CommandHandler}.success.count"; + public static string FaildCount = $"{CommandHandler}.failed.count"; + public static string ActiveCount = $"{CommandHandler}.active.count"; + public static string TotalExecutedCount = $"{CommandHandler}.total.count"; + public static string HandlerDuration = $"{CommandHandler}.duration"; + } + + public static class Queries + { + public static string Query = $"{ObservabilityConstant.InstrumentationName}.query"; + public static string QueryType = $"{Query}.type"; + public static string QueryHandler = $"{Query}.handler"; + public static string SuccessCount = $"{QueryHandler}.success.count"; + public static string FaildCount = $"{QueryHandler}.failed.count"; + public static string ActiveCount = $"{QueryHandler}.active.count"; + public static string TotalExecutedCount = $"{QueryHandler}.total.count"; + public static string HandlerDuration = $"{QueryHandler}.duration"; + } + + public static class Events + { + public static string Event = $"{ObservabilityConstant.InstrumentationName}.event"; + public static string EventType = $"{Event}.type"; + public static string EventHandler = $"{Event}.handler"; + public static string SuccessCount = $"{EventHandler}.success.count"; + public static string FaildCount = $"{EventHandler}.failed.count"; + public static string ActiveCount = $"{EventHandler}.active.count"; + public static string TotalExecutedCount = $"{EventHandler}.total.count"; + public static string HandlerDuration = $"{EventHandler}.duration"; + } + } + } +} diff --git a/src/Services/Booking/src/Booking.Api/appsettings.docker.json b/src/Services/Booking/src/Booking.Api/appsettings.docker.json index 8ad2a7d8..1ab06604 100644 --- a/src/Services/Booking/src/Booking.Api/appsettings.docker.json +++ b/src/Services/Booking/src/Booking.Api/appsettings.docker.json @@ -38,21 +38,11 @@ "LogOptions": { "Level": "information", "LogTemplate": "{Timestamp:HH:mm:ss} [{Level:u4}] {Message:lj}{NewLine}{Exception}", - "Elastic": { - "Enabled": true, - "ElasticServiceUrl": "elasticsearch:9200" - }, "File": { "Enabled": false, "Path": "logs/logs.txt", "Interval": "day" }, - "Sentry": { - "Enabled": false, - "Dsn": "", - "MinimumBreadcrumbLevel": "information", - "MinimumEventLevel":"error" - } }, "AllowedHosts": "*" } diff --git a/src/Services/Booking/src/Booking.Api/appsettings.json b/src/Services/Booking/src/Booking.Api/appsettings.json index d5084e8d..abfdd412 100644 --- a/src/Services/Booking/src/Booking.Api/appsettings.json +++ b/src/Services/Booking/src/Booking.Api/appsettings.json @@ -1,67 +1,78 @@ { - "AppOptions": { - "Name": "Booking-Service" - }, - "LogOptions": { - "Level": "information", - "LogTemplate": "{Timestamp:HH:mm:ss} [{Level:u4}] {Message:lj}{NewLine}{Exception}", - "Elastic": { - "Enabled": true, - "ElasticServiceUrl": "http://localhost:9200" + "AppOptions": { + "Name": "Booking-Service" }, - "File": { - "Enabled": false, - "Path": "logs/logs.txt", - "Interval": "day" + "LogOptions": { + "Level": "information", + "LogTemplate": "{Timestamp:HH:mm:ss} [{Level:u4}] {Message:lj}{NewLine}{Exception}", + "File": { + "Enabled": false, + "Path": "logs/logs.txt", + "Interval": "day" + }, }, - "Sentry": { - "Enabled": false, - "Dsn": "", - "MinimumBreadcrumbLevel": "information", - "MinimumEventLevel":"error" - } - }, - "Jwt": { - "Authority": "http://localhost:6005", - "Audience": "booking-api", - "RequireHttpsMetadata": false, - "MetadataAddress": "http://localhost:6005/.well-known/openid-configuration" - }, - "RabbitMqOptions": { - "HostName": "localhost", - "ExchangeName": "booking", - "UserName": "guest", - "Password": "guest", - "Port": 5672 - }, - "Grpc": { - "FlightAddress": "https://localhost:5003", - "PassengerAddress": "https://localhost:5012" - }, - "PolicyOptions": { - "Retry": { - "RetryCount": 3, - "SleepDuration": 1 - }, - "CircuitBreaker": { - "RetryCount": 5, - "BreakDuration" : 30 - } - }, - "EventStoreOptions": { - "ConnectionString": "esdb://localhost:2113?tls=false" - }, - "MongoOptions": { - "ConnectionString": "mongodb://localhost:27017", - "DatabaseName": "booking-db" - }, - "HealthOptions": { - "Enabled": false - }, - "PersistMessageOptions": { - "Interval": 30, - "Enabled": true, - "ConnectionString": "Server=localhost;Port=5432;Database=persist_message;User Id=postgres;Password=postgres;Include Error Detail=true" - }, - "AllowedHosts": "*" + "Jwt": { + "Authority": "http://localhost:6005", + "Audience": "booking-api", + "RequireHttpsMetadata": false, + "MetadataAddress": "http://localhost:6005/.well-known/openid-configuration" + }, + "RabbitMqOptions": { + "HostName": "localhost", + "ExchangeName": "booking", + "UserName": "guest", + "Password": "guest", + "Port": 5672 + }, + "Grpc": { + "FlightAddress": "https://localhost:5003", + "PassengerAddress": "https://localhost:5012" + }, + "PolicyOptions": { + "Retry": { + "RetryCount": 3, + "SleepDuration": 1 + }, + "CircuitBreaker": { + "RetryCount": 5, + "BreakDuration": 30 + } + }, + "EventStoreOptions": { + "ConnectionString": "esdb://localhost:2113?tls=false" + }, + "MongoOptions": { + "ConnectionString": "mongodb://localhost:27017", + "DatabaseName": "booking-db" + }, + "HealthOptions": { + "Enabled": false + }, + "PersistMessageOptions": { + "Interval": 30, + "Enabled": true, + "ConnectionString": "Server=localhost;Port=5432;Database=persist_message;User Id=postgres;Password=postgres;Include Error Detail=true" + }, + "ObservabilityOptions": { + "InstrumentationName": "booking_service", + "OTLPOptions": { + "OTLPGrpExporterEndpoint": "http://localhost:4317" + }, + "AspireDashboardOTLPOptions": { + "OTLPGrpExporterEndpoint": "http://localhost:4319" + }, + "ZipkinOptions": { + "HttpExporterEndpoint": "http://localhost:9411/api/v2/spans" + }, + "JaegerOptions": { + "OTLPGrpcExporterEndpoint": "http://localhost:14317", + "HttpExporterEndpoint": "http://localhost:14268/api/traces" + }, + "UsePrometheusExporter": true, + "UseOTLPExporter": true, + "UseAspireOTLPExporter": true, + "UseGrafanaExporter": false, + "ServiceName": "Booking Service" + }, + "AllowedHosts": "*" } diff --git a/src/Services/Booking/src/Booking/Extensions/Infrastructure/InfrastructureExtensions.cs b/src/Services/Booking/src/Booking/Extensions/Infrastructure/InfrastructureExtensions.cs index 18f8ba8d..919b6f5a 100644 --- a/src/Services/Booking/src/Booking/Extensions/Infrastructure/InfrastructureExtensions.cs +++ b/src/Services/Booking/src/Booking/Extensions/Infrastructure/InfrastructureExtensions.cs @@ -9,7 +9,7 @@ using BuildingBlocks.MassTransit; using BuildingBlocks.Mongo; using BuildingBlocks.OpenApi; -using BuildingBlocks.OpenTelemetry; +using BuildingBlocks.OpenTelemetryCollector; using BuildingBlocks.PersistMessageProcessor; using BuildingBlocks.ProblemDetails; using BuildingBlocks.Web; @@ -73,7 +73,7 @@ public static WebApplicationBuilder AddInfrastructure(this WebApplicationBuilder builder.Services.AddCustomMapster(typeof(BookingRoot).Assembly); builder.Services.AddCustomHealthCheck(); builder.Services.AddCustomMassTransit(env, typeof(BookingRoot).Assembly); - builder.Services.AddCustomOpenTelemetry(); + builder.AddCustomObservability(); builder.Services.AddTransient(); // ref: https://github.com/oskardudycz/EventSourcing.NetCore/tree/main/Sample/EventStoreDB/ECommerce @@ -91,7 +91,7 @@ public static WebApplication UseInfrastructure(this WebApplication app) var env = app.Environment; var appOptions = app.GetOptions(nameof(AppOptions)); - app.MapPrometheusScrapingEndpoint(); + app.UseCustomObservability(); app.UseCustomProblemDetails(); app.UseSerilogRequestLogging(options => diff --git a/src/Services/Flight/src/Flight.Api/appsettings.docker.json b/src/Services/Flight/src/Flight.Api/appsettings.docker.json index 5f211e82..d0daf016 100644 --- a/src/Services/Flight/src/Flight.Api/appsettings.docker.json +++ b/src/Services/Flight/src/Flight.Api/appsettings.docker.json @@ -3,20 +3,10 @@ "LogOptions": { "Level": "information", "LogTemplate": "{Timestamp:HH:mm:ss} [{Level:u4}] {Message:lj}{NewLine}{Exception}", - "Elastic": { - "Enabled": true, - "ElasticServiceUrl": "elasticsearch:9200" - }, "File": { "Enabled": false, "Path": "logs/logs.txt", "Interval": "day" - }, - "Sentry": { - "Enabled": false, - "Dsn": "", - "MinimumBreadcrumbLevel": "information", - "MinimumEventLevel": "error" } }, "PostgresOptions": { diff --git a/src/Services/Flight/src/Flight.Api/appsettings.json b/src/Services/Flight/src/Flight.Api/appsettings.json index 971e4d44..0f38f149 100644 --- a/src/Services/Flight/src/Flight.Api/appsettings.json +++ b/src/Services/Flight/src/Flight.Api/appsettings.json @@ -1,53 +1,64 @@ { - "AppOptions": { - "Name": "Flight-Service" - }, - "LogOptions": { - "Level": "information", - "LogTemplate": "{Timestamp:HH:mm:ss} [{Level:u4}] {Message:lj}{NewLine}{Exception}", - "Elastic": { - "Enabled": true, - "ElasticServiceUrl": "http://localhost:9200" - }, - "File": { - "Enabled": false, - "Path": "logs/logs.txt", - "Interval": "day" - }, - "Sentry": { - "Enabled": false, - "Dsn": "", - "MinimumBreadcrumbLevel": "information", - "MinimumEventLevel": "error" - } - }, - "PostgresOptions": { - "ConnectionString": "Server=localhost;Port=5432;Database=flight;User Id=postgres;Password=postgres;Include Error Detail=true" - }, - "MongoOptions": { - "ConnectionString": "mongodb://localhost:27017", - "DatabaseName": "flight-db" - }, - "Jwt": { - "Authority": "http://localhost:6005", - "Audience": "flight-api", - "RequireHttpsMetadata": false, - "MetadataAddress": "http://localhost:6005/.well-known/openid-configuration" - }, - "RabbitMqOptions": { - "HostName": "localhost", - "ExchangeName": "flight", - "UserName": "guest", - "Password": "guest", - "Port": 5672 - }, - "PersistMessageOptions": { - "Interval": 30, - "Enabled": true, - "ConnectionString": "Server=localhost;Port=5432;Database=persist_message;User Id=postgres;Password=postgres;Include Error Detail=true" - }, - "HealthOptions": { - "Enabled": false - }, - "AllowedHosts": "*" + "AppOptions": { + "Name": "Flight-Service" + }, + "LogOptions": { + "Level": "information", + "LogTemplate": "{Timestamp:HH:mm:ss} [{Level:u4}] {Message:lj}{NewLine}{Exception}", + "File": { + "Enabled": false, + "Path": "logs/logs.txt", + "Interval": "day" + } + }, + "PostgresOptions": { + "ConnectionString": "Server=localhost;Port=5432;Database=flight;User Id=postgres;Password=postgres;Include Error Detail=true" + }, + "MongoOptions": { + "ConnectionString": "mongodb://localhost:27017", + "DatabaseName": "flight-db" + }, + "Jwt": { + "Authority": "http://localhost:6005", + "Audience": "flight-api", + "RequireHttpsMetadata": false, + "MetadataAddress": "http://localhost:6005/.well-known/openid-configuration" + }, + "RabbitMqOptions": { + "HostName": "localhost", + "ExchangeName": "flight", + "UserName": "guest", + "Password": "guest", + "Port": 5672 + }, + "PersistMessageOptions": { + "Interval": 30, + "Enabled": true, + "ConnectionString": "Server=localhost;Port=5432;Database=persist_message;User Id=postgres;Password=postgres;Include Error Detail=true" + }, + "HealthOptions": { + "Enabled": false + }, + "ObservabilityOptions": { + "InstrumentationName": "flight_service", + "OTLPOptions": { + "OTLPGrpExporterEndpoint": "http://localhost:4317" + }, + "AspireDashboardOTLPOptions": { + "OTLPGrpExporterEndpoint": "http://localhost:4319" + }, + "ZipkinOptions": { + "HttpExporterEndpoint": "http://localhost:9411/api/v2/spans" + }, + "JaegerOptions": { + "OTLPGrpcExporterEndpoint": "http://localhost:14317", + "HttpExporterEndpoint": "http://localhost:14268/api/traces" + }, + "UsePrometheusExporter": true, + "UseOTLPExporter": true, + "UseAspireOTLPExporter": true, + "UseGrafanaExporter": false, + "ServiceName": "Flight Service" + }, + "AllowedHosts": "*" } diff --git a/src/Services/Flight/src/Flight/Extensions/Infrastructure/InfrastructureExtensions.cs b/src/Services/Flight/src/Flight/Extensions/Infrastructure/InfrastructureExtensions.cs index fd06f53b..d0c5f733 100644 --- a/src/Services/Flight/src/Flight/Extensions/Infrastructure/InfrastructureExtensions.cs +++ b/src/Services/Flight/src/Flight/Extensions/Infrastructure/InfrastructureExtensions.cs @@ -9,7 +9,7 @@ using BuildingBlocks.MassTransit; using BuildingBlocks.Mongo; using BuildingBlocks.OpenApi; -using BuildingBlocks.OpenTelemetry; +using BuildingBlocks.OpenTelemetryCollector; using BuildingBlocks.PersistMessageProcessor; using BuildingBlocks.ProblemDetails; using BuildingBlocks.Web; @@ -78,7 +78,7 @@ public static WebApplicationBuilder AddInfrastructure(this WebApplicationBuilder builder.Services.AddCustomMapster(typeof(FlightRoot).Assembly); builder.Services.AddHttpContextAccessor(); builder.Services.AddCustomMassTransit(env, typeof(FlightRoot).Assembly); - builder.Services.AddCustomOpenTelemetry(); + builder.AddCustomObservability(); builder.Services.AddCustomHealthCheck(); builder.Services.AddGrpc(options => @@ -97,7 +97,7 @@ public static WebApplication UseInfrastructure(this WebApplication app) var env = app.Environment; var appOptions = app.GetOptions(nameof(AppOptions)); - app.MapPrometheusScrapingEndpoint(); + app.UseCustomObservability(); app.UseCustomProblemDetails(); app.UseSerilogRequestLogging(options => diff --git a/src/Services/Flight/tests/IntegrationTest/Flight/Features/GetFlightByIdTests.cs b/src/Services/Flight/tests/IntegrationTest/Flight/Features/GetFlightByIdTests.cs index c0d1fd88..0df0609a 100644 --- a/src/Services/Flight/tests/IntegrationTest/Flight/Features/GetFlightByIdTests.cs +++ b/src/Services/Flight/tests/IntegrationTest/Flight/Features/GetFlightByIdTests.cs @@ -9,9 +9,7 @@ namespace Integration.Test.Flight.Features; -using global::Flight.Flights.Features.CreatingFlight.V1; using global::Flight.Flights.Features.GettingFlightById.V1; -using Thrift.Protocol; public class GetFlightByIdTests : FlightIntegrationTestBase { diff --git a/src/Services/Identity/src/Identity.Api/appsettings.docker.json b/src/Services/Identity/src/Identity.Api/appsettings.docker.json index d040ecb0..967ec5dc 100644 --- a/src/Services/Identity/src/Identity.Api/appsettings.docker.json +++ b/src/Services/Identity/src/Identity.Api/appsettings.docker.json @@ -21,20 +21,10 @@ "LogOptions": { "Level": "information", "LogTemplate": "{Timestamp:HH:mm:ss} [{Level:u4}] {Message:lj}{NewLine}{Exception}", - "Elastic": { - "Enabled": true, - "ElasticServiceUrl": "elasticsearch:9200" - }, "File": { "Enabled": false, "Path": "logs/logs.txt", "Interval": "day" - }, - "Sentry": { - "Enabled": false, - "Dsn": "", - "MinimumBreadcrumbLevel": "information", - "MinimumEventLevel":"error" } }, "AllowedHosts": "*" diff --git a/src/Services/Identity/src/Identity.Api/appsettings.json b/src/Services/Identity/src/Identity.Api/appsettings.json index 80f62ff9..f144a556 100644 --- a/src/Services/Identity/src/Identity.Api/appsettings.json +++ b/src/Services/Identity/src/Identity.Api/appsettings.json @@ -1,46 +1,57 @@ { - "AppOptions": { - "Name": "Identity-Service" - }, - "PostgresOptions": { - "ConnectionString": "Server=localhost;Port=5432;Database=identity;User Id=postgres;Password=postgres;Include Error Detail=true" - }, - "AuthOptions": { - "IssuerUri": "http://localhost:6005" - }, - "RabbitMqOptions": { - "HostName": "localhost", - "ExchangeName": "identity", - "UserName": "guest", - "Password": "guest", - "Port": 5672 - }, - "LogOptions": { - "Level": "information", - "LogTemplate": "{Timestamp:HH:mm:ss} [{Level:u4}] {Message:lj}{NewLine}{Exception}", - "Elastic": { - "Enabled": true, - "ElasticServiceUrl": "http://localhost:9200" - }, - "File": { - "Enabled": false, - "Path": "logs/logs.txt", - "Interval": "day" - }, - "Sentry": { - "Enabled": false, - "Dsn": "", - "MinimumBreadcrumbLevel": "information", - "MinimumEventLevel":"error" - } - }, - "HealthOptions": { - "Enabled": false - }, - "PersistMessageOptions": { - "Interval": 30, - "Enabled": true, - "ConnectionString": "Server=localhost;Port=5432;Database=persist_message;User Id=postgres;Password=postgres;Include Error Detail=true" - }, - "AllowedHosts": "*" + "AppOptions": { + "Name": "Identity-Service" + }, + "PostgresOptions": { + "ConnectionString": "Server=localhost;Port=5432;Database=identity;User Id=postgres;Password=postgres;Include Error Detail=true" + }, + "AuthOptions": { + "IssuerUri": "http://localhost:6005" + }, + "RabbitMqOptions": { + "HostName": "localhost", + "ExchangeName": "identity", + "UserName": "guest", + "Password": "guest", + "Port": 5672 + }, + "LogOptions": { + "Level": "information", + "LogTemplate": "{Timestamp:HH:mm:ss} [{Level:u4}] {Message:lj}{NewLine}{Exception}", + "File": { + "Enabled": false, + "Path": "logs/logs.txt", + "Interval": "day" + } + }, + "HealthOptions": { + "Enabled": false + }, + "PersistMessageOptions": { + "Interval": 30, + "Enabled": true, + "ConnectionString": "Server=localhost;Port=5432;Database=persist_message;User Id=postgres;Password=postgres;Include Error Detail=true" + }, + "ObservabilityOptions": { + "InstrumentationName": "identity_service", + "OTLPOptions": { + "OTLPGrpExporterEndpoint": "http://localhost:4317" + }, + "AspireDashboardOTLPOptions": { + "OTLPGrpExporterEndpoint": "http://localhost:4319" + }, + "ZipkinOptions": { + "HttpExporterEndpoint": "http://localhost:9411/api/v2/spans" + }, + "JaegerOptions": { + "OTLPGrpcExporterEndpoint": "http://localhost:14317", + "HttpExporterEndpoint": "http://localhost:14268/api/traces" + }, + "UsePrometheusExporter": true, + "UseOTLPExporter": true, + "UseAspireOTLPExporter": true, + "UseGrafanaExporter": false, + "ServiceName": "Identity Service" + }, + "AllowedHosts": "*" } diff --git a/src/Services/Identity/src/Identity/Extensions/Infrastructure/InfrastructureExtensions.cs b/src/Services/Identity/src/Identity/Extensions/Infrastructure/InfrastructureExtensions.cs index 8960e461..e58a084d 100644 --- a/src/Services/Identity/src/Identity/Extensions/Infrastructure/InfrastructureExtensions.cs +++ b/src/Services/Identity/src/Identity/Extensions/Infrastructure/InfrastructureExtensions.cs @@ -6,7 +6,7 @@ using BuildingBlocks.Mapster; using BuildingBlocks.MassTransit; using BuildingBlocks.OpenApi; -using BuildingBlocks.OpenTelemetry; +using BuildingBlocks.OpenTelemetryCollector; using BuildingBlocks.PersistMessageProcessor; using BuildingBlocks.ProblemDetails; using BuildingBlocks.Web; @@ -74,7 +74,7 @@ public static WebApplicationBuilder AddInfrastructure(this WebApplicationBuilder builder.Services.AddCustomHealthCheck(); builder.Services.AddCustomMassTransit(env, typeof(IdentityRoot).Assembly); - builder.Services.AddCustomOpenTelemetry(); + builder.AddCustomObservability(); builder.AddCustomIdentityServer(); @@ -93,7 +93,7 @@ public static WebApplication UseInfrastructure(this WebApplication app) var env = app.Environment; var appOptions = app.GetOptions(nameof(AppOptions)); - app.MapPrometheusScrapingEndpoint(); + app.UseCustomObservability(); app.UseForwardedHeaders(); diff --git a/src/Services/Passenger/src/Passenger.Api/appsettings.docker.json b/src/Services/Passenger/src/Passenger.Api/appsettings.docker.json index f55e0f81..5aa591db 100644 --- a/src/Services/Passenger/src/Passenger.Api/appsettings.docker.json +++ b/src/Services/Passenger/src/Passenger.Api/appsettings.docker.json @@ -28,20 +28,10 @@ "LogOptions": { "Level": "information", "LogTemplate": "{Timestamp:HH:mm:ss} [{Level:u4}] {Message:lj}{NewLine}{Exception}", - "Elastic": { - "Enabled": true, - "ElasticServiceUrl": "elasticsearch:9200" - }, "File": { "Enabled": false, "Path": "logs/logs.txt", "Interval": "day" - }, - "Sentry": { - "Enabled": false, - "Dsn": "", - "MinimumBreadcrumbLevel": "information", - "MinimumEventLevel":"error" } }, "AllowedHosts": "*" diff --git a/src/Services/Passenger/src/Passenger.Api/appsettings.json b/src/Services/Passenger/src/Passenger.Api/appsettings.json index 1ec2f4f2..951ae6dd 100644 --- a/src/Services/Passenger/src/Passenger.Api/appsettings.json +++ b/src/Services/Passenger/src/Passenger.Api/appsettings.json @@ -1,53 +1,64 @@ { - "AppOptions": { - "Name": "Passenger-Service" - }, - "PostgresOptions": { - "ConnectionString": "Server=localhost;Port=5432;Database=passenger;User Id=postgres;Password=postgres;Include Error Detail=true" - }, - "MongoOptions": { - "ConnectionString": "mongodb://localhost:27017", - "DatabaseName": "passenger-db" - }, - "Jwt": { - "Authority": "http://localhost:6005", - "Audience": "passenger-api", - "RequireHttpsMetadata": false, - "MetadataAddress": "http://localhost:6005/.well-known/openid-configuration" - }, - "RabbitMqOptions": { - "HostName": "localhost", - "ExchangeName": "passenger", - "UserName": "guest", - "Password": "guest", - "Port": 5672 - }, - "LogOptions": { - "Level": "information", - "LogTemplate": "{Timestamp:HH:mm:ss} [{Level:u4}] {Message:lj}{NewLine}{Exception}", - "Elastic": { - "Enabled": true, - "ElasticServiceUrl": "http://localhost:9200" - }, - "File": { - "Enabled": false, - "Path": "logs/logs.txt", - "Interval": "day" - }, - "Sentry": { - "Enabled": false, - "Dsn": "", - "MinimumBreadcrumbLevel": "information", - "MinimumEventLevel":"error" - } - }, - "HealthOptions": { - "Enabled": false - }, - "PersistMessageOptions": { - "Interval": 30, - "Enabled": true, - "ConnectionString": "Server=localhost;Port=5432;Database=persist_message;User Id=postgres;Password=postgres;Include Error Detail=true" - }, - "AllowedHosts": "*" + "AppOptions": { + "Name": "Passenger-Service" + }, + "PostgresOptions": { + "ConnectionString": "Server=localhost;Port=5432;Database=passenger;User Id=postgres;Password=postgres;Include Error Detail=true" + }, + "MongoOptions": { + "ConnectionString": "mongodb://localhost:27017", + "DatabaseName": "passenger-db" + }, + "Jwt": { + "Authority": "http://localhost:6005", + "Audience": "passenger-api", + "RequireHttpsMetadata": false, + "MetadataAddress": "http://localhost:6005/.well-known/openid-configuration" + }, + "RabbitMqOptions": { + "HostName": "localhost", + "ExchangeName": "passenger", + "UserName": "guest", + "Password": "guest", + "Port": 5672 + }, + "LogOptions": { + "Level": "information", + "LogTemplate": "{Timestamp:HH:mm:ss} [{Level:u4}] {Message:lj}{NewLine}{Exception}", + "File": { + "Enabled": false, + "Path": "logs/logs.txt", + "Interval": "day" + } + }, + "HealthOptions": { + "Enabled": false + }, + "PersistMessageOptions": { + "Interval": 30, + "Enabled": true, + "ConnectionString": "Server=localhost;Port=5432;Database=persist_message;User Id=postgres;Password=postgres;Include Error Detail=true" + }, + "ObservabilityOptions": { + "InstrumentationName": "passenger_service", + "OTLPOptions": { + "OTLPGrpExporterEndpoint": "http://localhost:4317" + }, + "AspireDashboardOTLPOptions": { + "OTLPGrpExporterEndpoint": "http://localhost:4319" + }, + "ZipkinOptions": { + "HttpExporterEndpoint": "http://localhost:9411/api/v2/spans" + }, + "JaegerOptions": { + "OTLPGrpcExporterEndpoint": "http://localhost:14317", + "HttpExporterEndpoint": "http://localhost:14268/api/traces" + }, + "UsePrometheusExporter": true, + "UseOTLPExporter": true, + "UseAspireOTLPExporter": true, + "UseGrafanaExporter": false, + "ServiceName": "Passenger Service" + }, + "AllowedHosts": "*" } diff --git a/src/Services/Passenger/src/Passenger/Extensions/Infrastructure/InfrastructureExtensions.cs b/src/Services/Passenger/src/Passenger/Extensions/Infrastructure/InfrastructureExtensions.cs index cbcadd8e..bd1169d8 100644 --- a/src/Services/Passenger/src/Passenger/Extensions/Infrastructure/InfrastructureExtensions.cs +++ b/src/Services/Passenger/src/Passenger/Extensions/Infrastructure/InfrastructureExtensions.cs @@ -9,7 +9,7 @@ using BuildingBlocks.MassTransit; using BuildingBlocks.Mongo; using BuildingBlocks.OpenApi; -using BuildingBlocks.OpenTelemetry; +using BuildingBlocks.OpenTelemetryCollector; using BuildingBlocks.PersistMessageProcessor; using BuildingBlocks.ProblemDetails; using BuildingBlocks.Web; @@ -26,7 +26,6 @@ namespace Passenger.Extensions.Infrastructure; - public static class InfrastructureExtensions { public static WebApplicationBuilder AddInfrastructure(this WebApplicationBuilder builder) @@ -38,27 +37,31 @@ public static WebApplicationBuilder AddInfrastructure(this WebApplicationBuilder builder.Services.AddScoped(); builder.Services.AddScoped(); - builder.Services.Configure(options => - { - options.SuppressModelStateInvalidFilter = true; - }); + builder.Services.Configure( + options => + { + options.SuppressModelStateInvalidFilter = true; + }); var appOptions = builder.Services.GetOptions(nameof(AppOptions)); Console.WriteLine(FiggleFonts.Standard.Render(appOptions.Name)); - builder.Services.AddRateLimiter(options => - { - options.GlobalLimiter = PartitionedRateLimiter.Create(httpContext => - RateLimitPartition.GetFixedWindowLimiter( - partitionKey: httpContext.User.Identity?.Name ?? httpContext.Request.Headers.Host.ToString(), - factory: partition => new FixedWindowRateLimiterOptions - { - AutoReplenishment = true, - PermitLimit = 10, - QueueLimit = 0, - Window = TimeSpan.FromMinutes(1) - })); - }); + builder.Services.AddRateLimiter( + options => + { + options.GlobalLimiter = PartitionedRateLimiter.Create( + httpContext => + RateLimitPartition.GetFixedWindowLimiter( + partitionKey: httpContext.User.Identity?.Name ?? + httpContext.Request.Headers.Host.ToString(), + factory: partition => new FixedWindowRateLimiterOptions + { + AutoReplenishment = true, + PermitLimit = 10, + QueueLimit = 0, + Window = TimeSpan.FromMinutes(1) + })); + }); builder.Services.AddPersistMessageProcessor(); builder.Services.AddCustomDbContext(); @@ -76,11 +79,13 @@ public static WebApplicationBuilder AddInfrastructure(this WebApplicationBuilder builder.Services.AddHttpContextAccessor(); builder.Services.AddCustomHealthCheck(); builder.Services.AddCustomMassTransit(env, typeof(PassengerRoot).Assembly); - builder.Services.AddCustomOpenTelemetry(); - builder.Services.AddGrpc(options => - { - options.Interceptors.Add(); - }); + builder.AddCustomObservability(); + + builder.Services.AddGrpc( + options => + { + options.Interceptors.Add(); + }); return builder; } @@ -91,13 +96,16 @@ public static WebApplication UseInfrastructure(this WebApplication app) var env = app.Environment; var appOptions = app.GetOptions(nameof(AppOptions)); - app.MapPrometheusScrapingEndpoint(); + app.UseCustomObservability(); app.UseCustomProblemDetails(); - app.UseSerilogRequestLogging(options => - { - options.EnrichDiagnosticContext = LogEnrichHelper.EnrichFromRequest; - }); + + app.UseSerilogRequestLogging( + options => + { + options.EnrichDiagnosticContext = LogEnrichHelper.EnrichFromRequest; + }); + app.UseCorrelationId(); app.UseMigration(); app.UseCustomHealthCheck(); diff --git a/src/Services/Passenger/tests/IntegrationTest/Passenger/Features/GetPassengerByIdTests.cs b/src/Services/Passenger/tests/IntegrationTest/Passenger/Features/GetPassengerByIdTests.cs index 3ca8e953..59ab0a58 100644 --- a/src/Services/Passenger/tests/IntegrationTest/Passenger/Features/GetPassengerByIdTests.cs +++ b/src/Services/Passenger/tests/IntegrationTest/Passenger/Features/GetPassengerByIdTests.cs @@ -10,8 +10,6 @@ namespace Integration.Test.Passenger.Features; using global::Passenger.Passengers.Features.GettingPassengerById.V1; -using Humanizer; -using Thrift.Protocol; public class GetPassengerByIdTests : PassengerIntegrationTestBase { From 9f284b3604c51cb5f9aa98a15bf109835519cc9d Mon Sep 17 00:00:00 2001 From: Meysam Hadeli <35596795+meysamhadeli@users.noreply.github.com> Date: Fri, 14 Feb 2025 01:23:38 +0330 Subject: [PATCH 2/3] feat: add full observability top of open telemetry collector --- .../Infrastructure/InfrastructureExtensions.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Services/Passenger/src/Passenger/Extensions/Infrastructure/InfrastructureExtensions.cs b/src/Services/Passenger/src/Passenger/Extensions/Infrastructure/InfrastructureExtensions.cs index bd1169d8..a7cbe48a 100644 --- a/src/Services/Passenger/src/Passenger/Extensions/Infrastructure/InfrastructureExtensions.cs +++ b/src/Services/Passenger/src/Passenger/Extensions/Infrastructure/InfrastructureExtensions.cs @@ -55,12 +55,12 @@ public static WebApplicationBuilder AddInfrastructure(this WebApplicationBuilder partitionKey: httpContext.User.Identity?.Name ?? httpContext.Request.Headers.Host.ToString(), factory: partition => new FixedWindowRateLimiterOptions - { - AutoReplenishment = true, - PermitLimit = 10, - QueueLimit = 0, - Window = TimeSpan.FromMinutes(1) - })); + { + AutoReplenishment = true, + PermitLimit = 10, + QueueLimit = 0, + Window = TimeSpan.FromMinutes(1) + })); }); builder.Services.AddPersistMessageProcessor(); From 3bf8733f5ed6d382843fea7da2dcc852d1269949 Mon Sep 17 00:00:00 2001 From: Meysam Hadeli <35596795+meysamhadeli@users.noreply.github.com> Date: Fri, 14 Feb 2025 01:59:57 +0330 Subject: [PATCH 3/3] fix: fix failed unit tests in flight --- .../CreateAircraftTests/CreateAircraftCommandHandlerTests.cs | 4 ---- .../CreateAirportTests/CreateAirportCommandHandlerTests.cs | 3 --- src/Services/Flight/tests/UnitTest/Common/DbContextFactory.cs | 2 -- src/Services/Flight/tests/UnitTest/Common/UnitTestFixture.cs | 1 - .../Flight/tests/UnitTest/Flight/FlightMappingTests.cs | 4 ---- .../UnitTest/Seat/Features/CreateSeatCommandHandlerTests.cs | 3 --- src/Services/Flight/tests/UnitTest/Seat/SeatMappingTests.cs | 2 -- 7 files changed, 19 deletions(-) diff --git a/src/Services/Flight/tests/UnitTest/Aircraft/Features/CreateAircraftTests/CreateAircraftCommandHandlerTests.cs b/src/Services/Flight/tests/UnitTest/Aircraft/Features/CreateAircraftTests/CreateAircraftCommandHandlerTests.cs index 6c299414..10c9f3ff 100644 --- a/src/Services/Flight/tests/UnitTest/Aircraft/Features/CreateAircraftTests/CreateAircraftCommandHandlerTests.cs +++ b/src/Services/Flight/tests/UnitTest/Aircraft/Features/CreateAircraftTests/CreateAircraftCommandHandlerTests.cs @@ -1,6 +1,3 @@ -using System; -using System.Threading; -using System.Threading.Tasks; using FluentAssertions; using Unit.Test.Common; using Unit.Test.Fakes; @@ -8,7 +5,6 @@ namespace Unit.Test.Aircraft.Features.CreateAircraftTests; -using global::Flight.Aircrafts.Dtos; using global::Flight.Aircrafts.Features.CreatingAircraft.V1; [Collection(nameof(UnitTestFixture))] diff --git a/src/Services/Flight/tests/UnitTest/Airport/Features/CreateAirportTests/CreateAirportCommandHandlerTests.cs b/src/Services/Flight/tests/UnitTest/Airport/Features/CreateAirportTests/CreateAirportCommandHandlerTests.cs index f430026c..d6a4137c 100644 --- a/src/Services/Flight/tests/UnitTest/Airport/Features/CreateAirportTests/CreateAirportCommandHandlerTests.cs +++ b/src/Services/Flight/tests/UnitTest/Airport/Features/CreateAirportTests/CreateAirportCommandHandlerTests.cs @@ -1,6 +1,3 @@ -using System; -using System.Threading; -using System.Threading.Tasks; using FluentAssertions; using Unit.Test.Common; using Unit.Test.Fakes; diff --git a/src/Services/Flight/tests/UnitTest/Common/DbContextFactory.cs b/src/Services/Flight/tests/UnitTest/Common/DbContextFactory.cs index b3d30c7e..f11753a8 100644 --- a/src/Services/Flight/tests/UnitTest/Common/DbContextFactory.cs +++ b/src/Services/Flight/tests/UnitTest/Common/DbContextFactory.cs @@ -1,5 +1,3 @@ -using System; -using System.Collections.Generic; using Flight.Data; using Flight.Flights.Enums; using Flight.Seats.Enums; diff --git a/src/Services/Flight/tests/UnitTest/Common/UnitTestFixture.cs b/src/Services/Flight/tests/UnitTest/Common/UnitTestFixture.cs index 57efaa65..102d7559 100644 --- a/src/Services/Flight/tests/UnitTest/Common/UnitTestFixture.cs +++ b/src/Services/Flight/tests/UnitTest/Common/UnitTestFixture.cs @@ -1,4 +1,3 @@ -using System; using Flight.Data; using MapsterMapper; using Xunit; diff --git a/src/Services/Flight/tests/UnitTest/Flight/FlightMappingTests.cs b/src/Services/Flight/tests/UnitTest/Flight/FlightMappingTests.cs index 78a69574..4c1e26e2 100644 --- a/src/Services/Flight/tests/UnitTest/Flight/FlightMappingTests.cs +++ b/src/Services/Flight/tests/UnitTest/Flight/FlightMappingTests.cs @@ -1,5 +1,3 @@ -using System; -using System.Collections.Generic; using Flight.Flights.Dtos; using MapsterMapper; using Unit.Test.Common; @@ -10,12 +8,10 @@ namespace Unit.Test.Flight; [Collection(nameof(UnitTestFixture))] public class FlightMappingTests { - private readonly UnitTestFixture _fixture; private readonly IMapper _mapper; public FlightMappingTests(UnitTestFixture fixture) { - _fixture = fixture; _mapper = fixture.Mapper; } diff --git a/src/Services/Flight/tests/UnitTest/Seat/Features/CreateSeatCommandHandlerTests.cs b/src/Services/Flight/tests/UnitTest/Seat/Features/CreateSeatCommandHandlerTests.cs index 5f85681b..ad9d85ae 100644 --- a/src/Services/Flight/tests/UnitTest/Seat/Features/CreateSeatCommandHandlerTests.cs +++ b/src/Services/Flight/tests/UnitTest/Seat/Features/CreateSeatCommandHandlerTests.cs @@ -1,6 +1,3 @@ -using System; -using System.Threading; -using System.Threading.Tasks; using FluentAssertions; using Unit.Test.Common; using Unit.Test.Fakes; diff --git a/src/Services/Flight/tests/UnitTest/Seat/SeatMappingTests.cs b/src/Services/Flight/tests/UnitTest/Seat/SeatMappingTests.cs index 1f749102..46526654 100644 --- a/src/Services/Flight/tests/UnitTest/Seat/SeatMappingTests.cs +++ b/src/Services/Flight/tests/UnitTest/Seat/SeatMappingTests.cs @@ -1,5 +1,3 @@ -using System; -using System.Collections.Generic; using Flight.Seats.Dtos; using MapsterMapper; using Unit.Test.Common;