feat: Add prometheus and grafana support

agents-api/agents_api/web.py

@@ -13,6 +13,7 @@
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse
 from litellm.exceptions import APIError
+from prometheus_fastapi_instrumentator import Instrumentator
 from pycozo.client import QueryException
 from scalar_fastapi import get_scalar_api_reference
 from temporalio.service import RPCError
@@ -99,6 +100,9 @@ def register_exceptions(app: FastAPI) -> None:
     root_path=api_prefix,
 )
 
+# Enable metrics
+Instrumentator().instrument(app).expose(app)
+
 # Create a new router for the docs
 scalar_router = APIRouter()
 

agents-api/pyproject.toml

@@ -38,6 +38,7 @@ scalar-fastapi = "^1.0.3"
 sse-starlette = "^2.1.3"
 anyio = "^4.4.0"
 python-box = {extras = ["toml"], version = "^7.2.0"}
+prometheus-fastapi-instrumentator = "^7.0.0"
 [tool.poetry.group.dev.dependencies]
 ipython = "^8.26.0"
 ruff = "^0.5.5"

docker-compose.yml

@@ -12,6 +12,8 @@ include:
   - ./scheduler/docker-compose.yml
   - ./llm-proxy/docker-compose.yml
   - ./integrations-service/docker-compose.yml
+  - ./prometheus/docker-compose.yml
+  - ./grafana/docker-compose.yml
 
   # TODO: Enable after testing
   # - ./monitoring/docker-compose.yml

grafana/docker-compose.yml

@@ -0,0 +1,23 @@
+name: grafana
+
+services:
+  grafana:
+    image: grafana/grafana
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD}
+      - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER}
+    container_name: grafana
+    ports:
+      - 3000:3000
+    volumes:
+      - grafana_data:/var/lib/grafana
+    profiles:
+      - multi-tenant
+
+volumes:
+  grafana_data:
+    driver: local
+    driver_opts:
+      o: bind
+      type: none
+      device: ./grafana_data

prometheus/docker-compose.yml

@@ -0,0 +1,17 @@
+name: prometheus
+
+services:
+  prometheus:
+    image: prom/prometheus
+    container_name: prometheus
+    profiles:
+      - multi-tenant
+
+    ports:
+      - 11000:9090
+
+    volumes:
+      - ./prometheus_data/prometheus.yml:/etc/prometheus/prometheus.yml
+
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'

prometheus/prometheus_data/prometheus.yml

@@ -0,0 +1,29 @@
+global:
+  scrape_interval: 15s
+  scrape_timeout: 10s
+  evaluation_interval: 15s
+alerting:
+  alertmanagers:
+  - follow_redirects: true
+    enable_http2: true
+    scheme: http
+    timeout: 10s
+    api_version: v2
+    static_configs:
+    - targets: []
+scrape_configs:
+- job_name: prometheus
+  honor_timestamps: true
+  scrape_interval: 15s
+  scrape_timeout: 10s
+  metrics_path: /metrics
+  scheme: http
+  follow_redirects: true
+  static_configs:
+  - targets:
+    - localhost:9090
+- job_name: 'fastapi'
+  scrape_interval: 10s
+  metrics_path: /metrics
+  static_configs:
+    - targets: ['agents-api:8080']