feat: add event stats for sidekiq and sq (#638)

Similar to Karafka's stats event, this adds the collected cluster based
stats as a wide event for Sidekiq and SolidQueue. The previous
aggregated metrics can be toggled on, while the new events can be
toggled off.

```
sidekiq:
  insights:
    events: false
solid_queue:
  insights:
    events: false

sidekiq:
  insights:
    metrics: true
solid_queue:
  insights:
    metrics: true
```

In addition, the polling interval has been decreased to 5 seconds to
allow for up to date readings.

lib/honeybadger/config/defaults.rb

@@ -344,41 +344,56 @@ class Boolean; end
         default: true,
         type: Boolean
       },
+      :'sidekiq.insights.enabled' => {
+        description: 'Enable automatic data collection for Sidekiq.',
+        default: true,
+        type: Boolean
+      },
+      :'sidekiq.insights.events' => {
+        description: 'Enable automatic event capturing for Sidekiq.',
+        default: true,
+        type: Boolean
+      },
+      :'sidekiq.insights.metrics' => {
+        description: 'Enable automatic metric data collection for Sidekiq.',
+        default: false,
+        type: Boolean
+      },
       :'sidekiq.insights.cluster_collection' => {
         description: 'Collect cluster based metrics for Sidekiq.',
         default: true,
         type: Boolean
       },
       :'sidekiq.insights.collection_interval' => {
         description: 'The frequency in which Sidekiq cluster metrics are sampled.',
-        default: 60,
+        default: 5,
         type: Integer
       },
-      :'solid_queue.insights.cluster_collection' => {
-        description: 'Collect cluster based metrics for SolidQueue.',
+      :'solid_queue.insights.enabled' => {
+        description: 'Enable automatic data collection for SolidQueue.',
         default: true,
         type: Boolean
       },
-      :'solid_queue.insights.collection_interval' => {
-        description: 'The frequency in which SolidQueue cluster metrics are sampled.',
-        default: 60,
-        type: Integer
-      },
-      :'sidekiq.insights.enabled' => {
-        description: 'Enable automatic data collection for Sidekiq.',
+      :'solid_queue.insights.events' => {
+        description: 'Enable automatic event capturing for SolidQueue.',
         default: true,
         type: Boolean
       },
-      :'sidekiq.insights.events' => {
-        description: 'Enable automatic event capturing for Sidekiq.',
-        default: true,
+      :'solid_queue.insights.metrics' => {
+        description: 'Enable automatic metric data collection for SolidQueue.',
+        default: false,
         type: Boolean
       },
-      :'sidekiq.insights.metrics' => {
-        description: 'Enable automatic metric data collection for Sidekiq.',
-        default: false,
+      :'solid_queue.insights.cluster_collection' => {
+        description: 'Collect cluster based metrics for SolidQueue.',
+        default: true,
         type: Boolean
       },
+      :'solid_queue.insights.collection_interval' => {
+        description: 'The frequency in which SolidQueue cluster metrics are sampled.',
+        default: 5,
+        type: Integer
+      },
       :'rails.insights.enabled' => {
         description: 'Enable automatic data collection for Ruby on Rails.',
         default: true,

lib/honeybadger/plugins/sidekiq.rb

@@ -167,47 +167,67 @@ def collect?
           end
         end
 
+        collect_sidekiq_stats = -> do
+          stats = ::Sidekiq::Stats.new
+          data = stats.as_json
+          data[:queues] = {}
+
+          ::Sidekiq::Queue.all.each do |queue|
+            data[:queues][queue.name] ||= {}
+            data[:queues][queue.name][:latency] = (queue.latency * 1000).ceil
+            data[:queues][queue.name][:depth] = queue.size
+          end
+
+          Hash.new(0).tap do |busy_counts|
+            ::Sidekiq::Workers.new.each do |_pid, _tid, work|
+              payload = work.respond_to?(:payload) ? work.payload : work["payload"]
+              payload = JSON.parse(payload) if payload.is_a?(String)
+              busy_counts[payload["queue"]] += 1
+            end
+          end.each do |queue_name, busy_count|
+            data[:queues][queue_name] ||= {}
+            data[:queues][queue_name][:busy] = busy_count
+          end
+
+          processes = ::Sidekiq::ProcessSet.new.to_enum(:each).to_a
+          data[:capacity] = processes.map { |process| process["concurrency"] }.sum
+
+          process_utilizations = processes.map do |process|
+            next unless process["concurrency"].to_f > 0
+            process["busy"] / process["concurrency"].to_f
+          end.compact
+
+          if process_utilizations.any?
+            utilization = process_utilizations.sum / process_utilizations.length.to_f
+            data[:utilization] = utilization
+          end
+
+          data
+        end
+
         collect do
           if config.cluster_collection?(:sidekiq) && (leader_checker.nil? || leader_checker.collect?)
-            metric_source 'sidekiq'
-
-            stats = ::Sidekiq::Stats.new
-
-            gauge 'active_workers', ->{ stats.workers_size }
-            gauge 'active_processes', ->{ stats.processes_size }
-            gauge 'jobs_processed', ->{ stats.processed }
-            gauge 'jobs_failed', ->{ stats.failed }
-            gauge 'jobs_scheduled', ->{ stats.scheduled_size }
-            gauge 'jobs_enqueued', ->{ stats.enqueued }
-            gauge 'jobs_dead', ->{ stats.dead_size }
-            gauge 'jobs_retry', ->{ stats.retry_size }
-
-            ::Sidekiq::Queue.all.each do |queue|
-              gauge 'queue_latency', { queue: queue.name }, ->{ (queue.latency * 1000).ceil }
-              gauge 'queue_depth', { queue: queue.name }, ->{ queue.size }
-            end
+            stats = collect_sidekiq_stats.call
 
-            Hash.new(0).tap do |busy_counts|
-              ::Sidekiq::Workers.new.each do |_pid, _tid, work|
-                payload = work.respond_to?(:payload) ? work.payload : work["payload"]
-                payload = JSON.parse(payload) if payload.is_a?(String)
-                busy_counts[payload["queue"]] += 1
-              end
-            end.each do |queue_name, busy_count|
-              gauge 'queue_busy', { queue: queue_name }, ->{ busy_count }
+            if Honeybadger.config.load_plugin_insights_events?(:sidekiq)
+              Honeybadger.event('stats.sidekiq', stats.except('stats').merge(stats['stats']))
             end
 
-            processes = ::Sidekiq::ProcessSet.new.to_enum(:each).to_a
-            gauge 'capacity', ->{ processes.map { |process| process["concurrency"] }.sum }
+            if Honeybadger.config.load_plugin_insights_metrics?(:sidekiq)
+              metric_source 'sidekiq'
+
+              stats['stats'].each do |name, value|
+                gauge name, value: value
+              end
 
-            process_utilizations = processes.map do |process|
-              next unless process["concurrency"].to_f > 0
-              process["busy"] / process["concurrency"].to_f
-            end.compact
+              stats[:queues].each do |queue_name, data|
+                data.each do |key, value|
+                  gauge "queue_#{key}", queue: queue_name, value: value
+                end
+              end
 
-            if process_utilizations.any?
-              utilization = process_utilizations.sum / process_utilizations.length.to_f
-              gauge 'utilization', ->{ utilization }
+              gauge 'capacity', value: stats[:capacity] if stats[:capacity]
+              gauge 'utilization', value: stats[:utilization] if stats[:utilization]
             end
           end
         end

lib/honeybadger/plugins/solid_queue.rb

@@ -4,20 +4,46 @@ module SolidQueue
       Plugin.register :solid_queue do
         requirement { config.load_plugin_insights?(:solid_queue) && defined?(::SolidQueue) }
 
+        collect_solid_queue_stats = -> do
+          data = {}
+          data[:stats] = {
+            jobs_in_progress: ::SolidQueue::ClaimedExecution.count,
+            jobs_blocked: ::SolidQueue::BlockedExecution.count,
+            jobs_failed: ::SolidQueue::FailedExecution.count,
+            jobs_scheduled: ::SolidQueue::ScheduledExecution.count,
+            jobs_processed: ::SolidQueue::Job.where.not(finished_at: nil).count,
+            active_workers: ::SolidQueue::Process.where(kind: "Worker").count,
+            active_dispatchers: ::SolidQueue::Process.where(kind: "Dispatcher").count
+          }
+
+          data[:queues] = {}
+
+          ::SolidQueue::Queue.all.each do |queue|
+            data[:queues][queue.name] = { depth: queue.size }
+          end
+
+          data
+        end
+
         collect do
+          stats = collect_solid_queue_stats.call
+
           if config.cluster_collection?(:solid_queue)
-            metric_source 'solid_queue'
-
-            gauge 'jobs_in_progress', ->{ ::SolidQueue::ClaimedExecution.count }
-            gauge 'jobs_blocked', ->{ ::SolidQueue::BlockedExecution.count }
-            gauge 'jobs_failed', ->{ ::SolidQueue::FailedExecution.count }
-            gauge 'jobs_scheduled', ->{ ::SolidQueue::ScheduledExecution.count }
-            gauge 'jobs_processed', ->{ ::SolidQueue::Job.where.not(finished_at: nil).count }
-            gauge 'active_workers', ->{ ::SolidQueue::Process.where(kind: "Worker").count }
-            gauge 'active_dispatchers', ->{ ::SolidQueue::Process.where(kind: "Dispatcher").count }
-
-            ::SolidQueue::Queue.all.each do |queue|
-              gauge 'queue_depth', { queue: queue.name }, ->{ queue.size }
+            if Honeybadger.config.load_plugin_insights_events?(:solid_queue)
+              Honeybadger.event('stats.solid_queue', stats.except(:stats).merge(stats[:stats]))
+            end
+
+            if Honeybadger.config.load_plugin_insights_metrics?(:solid_queue)
+              metric_source 'solid_queue'
+              stats[:stats].each do |stat_name, value|
+                gauge stat_name, value: value
+              end
+
+              stats[:queues].each do |queue_name, data|
+                data.each do |key, value|
+                  gauge "queue_#{key}", queue: queue_name, value: value
+                end
+              end
             end
           end
         end

spec/unit/honeybadger/plugins/sidekiq_spec.rb

@@ -197,6 +197,9 @@ def scheduled_size; end
           def enqueued; end
           def dead_size; end
           def retry_size; end
+          def as_json
+            {}
+          end
         end
       end