Updated kafka integration to include all stats (DataDog#498)

* Updated kafka integration to include all stats https://github.com/DataDog/integrations-core/blob/master/kafka/datadog_checks/kafka/data/conf.yaml.example * Updated comment * Tags can no longer be used for not being present Tags are part of the static portion of the template so let's look for another conditional parameter.
cegeka · Mar 14, 2019 · a7830b7 · a7830b7
1 parent 7de3c7f
commit a7830b7
Show file tree

Hide file tree

Showing 2 changed files with 249 additions and 1 deletion.
diff --git a/spec/classes/datadog_agent_integrations_kafka_spec.rb b/spec/classes/datadog_agent_integrations_kafka_spec.rb
@@ -35,7 +35,7 @@
 
       context 'with default parameters' do
         it { should contain_file(conf_file).with_content(%r{- host: localhost\s+port: 9999}) }
-        it { should contain_file(conf_file).without_content(%r{tags:}) }
+        it { should contain_file(conf_file).without_content(%r{user:}) }
       end
 
       context 'with one kafka broker' do

diff --git a/templates/agent-conf.d/kafka.yaml.erb b/templates/agent-conf.d/kafka.yaml.erb
@@ -118,6 +118,131 @@ init_config:
             metric_type: gauge
             alias: kafka.producer.io_wait
 
+    #
+    # Producers (v0.11.x)
+    #
+    - include:
+        domain: 'kafka.producer'
+        bean_regex: 'kafka.producer:type=producer-metrics,client-id=([-.\w]+)'
+        attribute:
+          - waiting-threads:
+              metric_type: gauge
+              alias: kafka.producer.waiting_threads
+          - buffer-total-bytes:
+              metric_type: gauge
+              alias: kafka.producer.buffer_bytes_total
+          - buffer-available-bytes:
+              metric_type: gauge
+              alias: kafka.producer.available_buffer_bytes
+          - bufferpool-wait-time:
+              metric_type: gauge
+              alias: kafka.producer.bufferpool_wait_time
+          - batch-size-avg:
+              metric_type: gauge
+              alias: kafka.producer.batch_size_avg
+          - batch-size-max:
+              metric_type: gauge
+              alias: kafka.producer.batch_size_max
+          - compression-rate-avg:
+              metric_type: rate
+              alias: kafka.producer.compression_rate_avg
+          - record-queue-time-avg:
+              metric_type: gauge
+              alias: kafka.producer.record_queue_time_avg
+          - record-queue-time-max:
+              metric_type: gauge
+              alias: kafka.producer.record_queue_time_max
+          - request-latency-avg:
+              metric_type: gauge
+              alias: kafka.producer.request_latency_avg
+          - request-latency-max:
+              metric_type: gauge
+              alias: kafka.producer.request_latency_max
+          - record-send-rate:
+              metric_type: gauge
+              alias: kafka.producer.records_send_rate
+          - records-per-request-avg:
+              metric_type: gauge
+              alias: kafka.producer.records_per_request
+          - record-retry-rate:
+              metric_type: gauge
+              alias: kafka.producer.record_retry_rate
+          - record-error-rate:
+              metric_type: gauge
+              alias: kafka.producer.record_error_rate
+          - record-size-max:
+              metric_type: gauge
+              alias: kafka.producer.record_size_max
+          - record-size-avg:
+              metric_type: gauge
+              alias: kafka.producer.record_size_avg
+          - requests-in-flight:
+              metric_type: gauge
+              alias: kafka.producer.requests_in_flight
+          - metadata-age:
+              metric_type: gauge
+              alias: kafka.producer.metadata_age
+          - produce-throttle-time-max:
+              metric_type: gauge
+              alias: kafka.producer.throttle_time_max
+          - produce-throttle-time-avg:
+              metric_type: gauge
+              alias: kafka.producer.throttle_time_avg
+
+    #
+    # Producers: Per Topic Metrics
+    #
+    - include:
+        domain: 'kafka.producer'
+        bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
+        attribute:
+          byte-rate:
+            metric_type: gauge
+            alias: kafka.producer.bytes_out
+        tags:
+          client: $1
+          topic: $2
+    - include:
+        domain: 'kafka.producer'
+        bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
+        attribute:
+          record-send-rate:
+            metric_type: gauge
+            alias: kafka.producer.record_send_rate
+        tags:
+          client: $1
+          topic: $2
+    - include:
+        domain: 'kafka.producer'
+        bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
+        attribute:
+          compression-rate:
+            metric_type: gauge
+            alias: kafka.producer.compression_rate
+        tags:
+          client: $1
+          topic: $2
+    - include:
+        domain: 'kafka.producer'
+        bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
+        attribute:
+          record-retry-rate:
+            metric_type: gauge
+            alias: kafka.producer.record_retry_rate
+        tags:
+          client: $1
+          topic: $2
+    - include:
+        domain: 'kafka.producer'
+        bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
+        attribute:
+          record-error-rate:
+            metric_type: gauge
+            alias: kafka.producer.record_error_rate
+        tags:
+          client: $1
+          topic: $2
+
     #
     # Consumers (only v0.8.2.x)
     #
@@ -184,6 +309,61 @@ init_config:
             metric_type: gauge
             alias: kafka.consumer.messages_in
 
+    #
+    # Consumers: Per Topic Metrics
+    #
+
+    - include:
+        domain: 'kafka.consumer'
+        bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
+        attribute:
+          fetch-size-avg:
+            metric_type: gauge
+            alias: kafka.consumer.fetch_size_avg
+        tags:
+          client: $1
+          topic: $2
+    - include:
+        domain: 'kafka.consumer'
+        bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
+        attribute:
+          fetch-size-max:
+            metric_type: gauge
+            alias: kafka.consumer.fetch_size_max
+        tags:
+          client: $1
+          topic: $2
+    - include:
+        domain: 'kafka.consumer'
+        bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
+        attribute:
+          bytes-consumed-rate:
+            metric_type: gauge
+            alias: kafka.consumer.bytes_consumed
+        tags:
+          client: $1
+          topic: $2
+    - include:
+        domain: 'kafka.consumer'
+        bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
+        attribute:
+          records-per-request-avg:
+            metric_type: gauge
+            alias: kafka.consumer.records_per_request_avg
+        tags:
+          client: $1
+          topic: $2
+    - include:
+        domain: 'kafka.consumer'
+        bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
+        attribute:
+          records-consumed-rate:
+            metric_type: gauge
+            alias: kafka.consumer.records_consumed
+        tags:
+          client: $1
+          topic: $2
+
     #
     # Aggregate cluster stats
     #
@@ -216,6 +396,38 @@ init_config:
             metric_type: rate
             alias: kafka.net.bytes_rejected.rate
 
+    #
+    # Brokers: Per Topic Metrics
+    #
+    - include:
+        domain: 'kafka.server'
+        bean_regex: 'kafka\.server:type=BrokerTopicMetrics,name=BytesOutPerSec,topic=.*'
+        attribute:
+          Count:
+            metric_type: rate
+            alias: kafka.topics.net.bytes_out.rate
+    - include:
+        domain: 'kafka.server'
+        bean_regex: 'kafka\.server:type=BrokerTopicMetrics,name=BytesInPerSec,topic=.*'
+        attribute:
+          Count:
+            metric_type: rate
+            alias: kafka.topics.net.bytes_in.rate
+    - include:
+        domain: 'kafka.server'
+        bean_regex: 'kafka\.server:type=BrokerTopicMetrics,name=MessagesInPerSec,topic=.*'
+        attribute:
+          Count:
+            metric_type: rate
+            alias: kafka.topics.messages_in.rate
+    - include:
+        domain: 'kafka.server'
+        bean_regex: 'kafka\.server:type=BrokerTopicMetrics,name=BytesRejectedPerSec,topic=.*'
+        attribute:
+          Count:
+            metric_type: rate
+            alias: kafka.topics.net.bytes_rejected.rate
+
     #
     # Request timings
     #
@@ -314,13 +526,31 @@ init_config:
           99thPercentile:
             metric_type: gauge
             alias: kafka.request.offsets.time.99percentile
+    - include:
+        domain: 'kafka.network'
+        bean: 'kafka.network:type=RequestChannel,name=RequestQueueSize'
+        attribute:
+          Value:
+            metric_type: gauge
+            alias: kafka.request.channel.queue.size
+    - include:
+        domain: 'kafka.network'
+        bean: 'kafka.network:type=SocketServer,name=NetworkProcessorAvgIdlePercent'
+        attribute:
+          Value:
+            metric_type: gauge
+            alias: kafka.net.handler.avg.idle.pct.rate
     - include:
         domain: 'kafka.server'
         bean: 'kafka.server:type=KafkaRequestHandlerPool,name=RequestHandlerAvgIdlePercent'
         attribute:
           OneMinuteRate:
             metric_type: gauge
             alias: kafka.request.handler.avg.idle.pct.rate
+
+    #
+    # Request Purgatory (only v0.8.2.x)
+    #
     - include:
         domain: 'kafka.server'
         bean: 'kafka.server:type=ProducerRequestPurgatory,name=PurgatorySize'
@@ -336,6 +566,24 @@ init_config:
             metric_type: gauge
             alias: kafka.request.fetch_request_purgatory.size
 
+    #
+    # Request Purgatory (v0.9.0.x onwards)
+    #
+    - include:
+        domain: 'kafka.server'
+        bean: 'kafka.server:type=DelayedOperationPurgatory,name=PurgatorySize,delayedOperation=Produce'
+        attribute:
+          Value:
+            metric_type: gauge
+            alias: kafka.request.producer_request_purgatory.size
+    - include:
+        domain: 'kafka.server'
+        bean: 'kafka.server:type=DelayedOperationPurgatory,name=PurgatorySize,delayedOperation=Fetch'
+        attribute:
+          Value:
+            metric_type: gauge
+            alias: kafka.request.fetch_request_purgatory.size
+
     #
     # Replication stats
     #