From a5e9b35314c6bae14a2c447505cbc64e04f846d5 Mon Sep 17 00:00:00 2001 From: Socheat Sok Date: Wed, 12 Jun 2024 16:31:34 +0700 Subject: [PATCH] Refactor rootfs/dockerswarm/dockerswarm-services.yml -> rootfs/dockerswarm/dockerswarm-endpoints-nodeport.yml --- .../dockerswarm-endpoints-ingresses.yml | 110 ++++++++++++++++++ ...yml => dockerswarm-endpoints-nodeport.yml} | 25 +++- test/docker-stack.yml | 28 +++-- 3 files changed, 146 insertions(+), 17 deletions(-) create mode 100644 rootfs/dockerswarm/dockerswarm-endpoints-ingresses.yml rename rootfs/dockerswarm/{dockerswarm-services.yml => dockerswarm-endpoints-nodeport.yml} (83%) diff --git a/rootfs/dockerswarm/dockerswarm-endpoints-ingresses.yml b/rootfs/dockerswarm/dockerswarm-endpoints-ingresses.yml new file mode 100644 index 0000000..1d1db8c --- /dev/null +++ b/rootfs/dockerswarm/dockerswarm-endpoints-ingresses.yml @@ -0,0 +1,110 @@ +# The scrape config for probing services via the Blackbox Exporter. +# +# The relabeling allows the actual service scrape endpoint to be configured +# for all or only some services. +# +# Annotations: +# io.prometheus.probe_enabled= +# io.prometheus.probe_disabled= +# +# io.prometheus.dockerswarm-endpoints.should_be_probed= +scrape_configs: + + - job_name: 'dockerswarm-endpoints-ingresses' + scrape_interval: 15s + + metrics_path: /probe + params: + module: [icmp_ttl5] + + dockerswarm_sd_configs: + - host: unix:///var/run/docker.sock + role: services + refresh_interval: 10s + + relabel_configs: + # ================================================================================ + # Keep or drop targets with the following rules + # ================================================================================ + + # io.prometheus.dockerswarm-endpoints.should_be_probed= + - source_labels: + - __meta_dockerswarm_service_label_io_prometheus_dockerswarm_endpoints_should_be_probed + regex: 'false' + action: drop + + # io.prometheus.probe_enabled= + - source_labels: + - __meta_dockerswarm_service_label_io_prometheus_probe_enabled + regex: 'false' + action: drop + # io.prometheus.probe_disabled= + - source_labels: + - __meta_dockerswarm_service_label_io_prometheus_probe_disabled + regex: 'false' + action: drop + + # Keep only tasks connected to the "host" network + - source_labels: + - __meta_dockerswarm_network_ingress + regex: 'true' + action: keep + - source_labels: + - __meta_dockerswarm_service_endpoint_port_publish_mode + regex: 'ingress' + action: keep + + # ================================================================================ + # Override prometheus and blackbox internal labels + # ================================================================================ + - source_labels: + - __address__ + target_label: __tmp_target + regex: ([^:]+)(?::\d+)? + replacement: $1 + - source_labels: [__address__] + target_label: __tmp_address + - source_labels: [__tmp_target] + target_label: __param_target + - target_label: __address__ + replacement: blackbox-exporter.svc.cluster.local:9115 + - source_labels: [__tmp_address] + target_label: instance + + # Drop all labels starting with "io.prometheus." + - action: labeldrop + regex: __meta_(dockerswarm_service_label_io_prometheus_.+) + + # ================================================================================ + # Label mapping + # ================================================================================ + - action: labelmap + regex: __meta_(dockerswarm_.+) + + # ================================================================================ + # Kubernetes compatible relabeling + # - namespace + # - deployment + # - pod + # ================================================================================ + # Set Kubernetes's Namespace with "com.docker.stack.namespace" label + - source_labels: + - __meta_dockerswarm_service_label_com_docker_stack_namespace + target_label: namespace + + # Set Kubernetes's Deployment with "com.docker.stack.namespace" label + - source_labels: + - __meta_dockerswarm_service_label_com_docker_stack_namespace + target_label: deployment + + # Set Kubernetes' Service Name with Docker Swarm's Service Name + - source_labels: + - __meta_dockerswarm_service_name + target_label: service + separator: '.' + + # Set Kubernetes's Pod Name with Docker Swarm's Service Name + - source_labels: + - __meta_dockerswarm_service_name + target_label: pod + separator: '.' diff --git a/rootfs/dockerswarm/dockerswarm-services.yml b/rootfs/dockerswarm/dockerswarm-endpoints-nodeport.yml similarity index 83% rename from rootfs/dockerswarm/dockerswarm-services.yml rename to rootfs/dockerswarm/dockerswarm-endpoints-nodeport.yml index 4784e01..7acb7ca 100644 --- a/rootfs/dockerswarm/dockerswarm-services.yml +++ b/rootfs/dockerswarm/dockerswarm-endpoints-nodeport.yml @@ -10,12 +10,12 @@ # io.prometheus.dockerswarm-services.should_be_probed= scrape_configs: - - job_name: 'dockerswarm-services' + - job_name: 'dockerswarm-endpoints-nodeport' scrape_interval: 15s metrics_path: /probe params: - module: [http_2xx] + module: [icmp_ttl5] dockerswarm_sd_configs: - host: unix:///var/run/docker.sock @@ -29,7 +29,7 @@ scrape_configs: # io.prometheus.dockerswarm-services.should_be_probed= - source_labels: - - __meta_dockerswarm_service_label_io_prometheus_dockerswarm_services_should_be_probed + - __meta_dockerswarm_service_label_io_prometheus_dockerswarm_endpoints_should_be_probed regex: 'false' action: drop @@ -44,7 +44,15 @@ scrape_configs: regex: 'false' action: drop - # Keep only tasks connected to the "dockerswarm_metrics" network + # Keep only tasks connected to the "ingress" network + - source_labels: + - __meta_dockerswarm_network_ingress + regex: 'true' + action: drop + - source_labels: + - __meta_dockerswarm_service_endpoint_port_publish_mode + regex: 'host' + action: keep - source_labels: - __meta_dockerswarm_network_name regex: (^dockerswarm_metrics$) @@ -53,11 +61,18 @@ scrape_configs: # ================================================================================ # Override prometheus and blackbox internal labels # ================================================================================ + - source_labels: + - __address__ + target_label: __tmp_target + regex: ([^:]+)(?::\d+)? + replacement: $1 - source_labels: [__address__] + target_label: __tmp_address + - source_labels: [__tmp_target] target_label: __param_target - target_label: __address__ replacement: blackbox-exporter.svc.cluster.local:9115 - - source_labels: [__param_target] + - source_labels: [__tmp_address] target_label: instance # Drop all labels starting with "io.prometheus." diff --git a/test/docker-stack.yml b/test/docker-stack.yml index 1f9cefc..c1e96c1 100644 --- a/test/docker-stack.yml +++ b/test/docker-stack.yml @@ -66,7 +66,7 @@ services: labels: io.prometheus.role: "prometheus" io.prometheus.dockerswarm-tasks.should_be_scraped: "false" - io.prometheus.dockerswarm-services.should_be_probed: "false" + io.prometheus.dockerswarm-endpoints.should_be_probed: "false" logging: *x-default-logging environment: - DOCKERSWARM_SERVICE_ID={{.Service.ID}} @@ -97,6 +97,7 @@ services: blackbox-exporter: image: prom/blackbox-exporter:latest + command: --config.file=/etc/blackbox_exporter/config.yml --log.level=debug deploy: replicas: 1 resources: *x-exporter-resources-constraints @@ -104,8 +105,11 @@ services: io.prometheus.enabled: "true" io.prometheus.job_name: "blackbox-exporter" io.prometheus.scrape_port: "9115" - io.prometheus.dockerswarm-services.should_be_probed: "false" + io.prometheus.dockerswarm-endpoints.should_be_probed: "false" logging: *x-default-logging + ports: + - published: 9115 + target: 9115 hostname: blackbox-exporter.svc.cluster.local networks: dockerswarm_metrics: @@ -124,12 +128,12 @@ services: labels: io.prometheus.role: "node-exporter" io.prometheus.dockerswarm-tasks.should_be_scraped: "false" - io.prometheus.dockerswarm-services.should_be_probed: "false" + io.prometheus.dockerswarm-endpoints.should_be_probed: "false" logging: *x-default-logging - # ports: - # - published: 9100 - # target: 9100 - # mode: host + ports: + - published: 9100 + target: 9100 + mode: host networks: prometheus_exporter: hostname: node-exporter.{{.Node.ID}}.cluster.local @@ -153,12 +157,12 @@ services: labels: io.prometheus.role: "cadvisor" io.prometheus.dockerswarm-tasks.should_be_scraped: "false" - io.prometheus.dockerswarm-services.should_be_probed: "false" + io.prometheus.dockerswarm-endpoints.should_be_probed: "false" logging: *x-default-logging - # ports: - # - published: 8080 - # target: 8080 - # mode: host + ports: + - published: 8080 + target: 8080 + mode: host networks: prometheus_exporter: hostname: cadvisor.{{.Node.ID}}.cluster.local