agent: Add runner panics metrics (#180)

Typically, panics would be visible in other ways, like K8s events. But
because the autoscaler-agent isolates panics just to the threads
handling a single VM, these can go unnoticed unless we do something
about it.

pkg/agent/prommetrics.go

@@ -9,6 +9,7 @@ type PromMetrics struct {
 	schedulerRequests         *prometheus.CounterVec
 	informantRequestsOutbound *prometheus.CounterVec
 	informantRequestsInbound  *prometheus.CounterVec
+	runnerThreadPanics        prometheus.Counter
 }
 
 func makePrometheusParts(globalstate *agentState) (PromMetrics, *prometheus.Registry) {
@@ -33,6 +34,37 @@ func makePrometheusParts(globalstate *agentState) (PromMetrics, *prometheus.Regi
 		},
 		[]string{"endpoint", "code"},
 	)
+	runnerThreadPanics := prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "autoscaling_agent_runner_thread_panics_total",
+			Help: "Number of panics from autoscaler-agent per-VM runner threads",
+		},
+	)
+	totalPanickedVMs := prometheus.NewGaugeFunc(
+		prometheus.GaugeOpts{
+			Name: "autoscaling_panicked_vm_runners_current",
+			Help: "Number of VMs whose per-VM runner has panicked (and not restarted)",
+		},
+		func() float64 {
+			globalstate.lock.Lock()
+			defer globalstate.lock.Unlock()
+
+			count := 0
+
+			for _, p := range globalstate.pods {
+				func() {
+					p.status.mu.Lock()
+					defer p.status.mu.Unlock()
+
+					if p.status.panicked {
+						count += 1
+					}
+				}()
+			}
+
+			return float64(count)
+		},
+	)
 	totalVMs := prometheus.NewGaugeFunc(
 		prometheus.GaugeOpts{
 			Name: "autoscaling_agent_tracked_vms_current",
@@ -73,6 +105,8 @@ func makePrometheusParts(globalstate *agentState) (PromMetrics, *prometheus.Regi
 		schedulerRequests,
 		informantRequestsOutbound,
 		informantRequestsInbound,
+		runnerThreadPanics,
+		totalPanickedVMs,
 		totalVMs,
 		totalVMsWithUnhealthyInformants,
 	)
@@ -81,5 +115,6 @@ func makePrometheusParts(globalstate *agentState) (PromMetrics, *prometheus.Regi
 		schedulerRequests:         schedulerRequests,
 		informantRequestsOutbound: informantRequestsOutbound,
 		informantRequestsInbound:  informantRequestsInbound,
+		runnerThreadPanics:        runnerThreadPanics,
 	}, reg
 }

pkg/agent/runner.go

@@ -401,6 +401,8 @@ func (r *Runner) spawnBackgroundWorker(ctx context.Context, name string, f func(
 			r.backgroundWorkerCount.Add(-1)
 
 			if v := recover(); v != nil {
+				r.global.metrics.runnerThreadPanics.Inc()
+
 				err := fmt.Errorf("background worker %q panicked: %v", name, v)
 				r.logger.Errorf("%s", err)
 				// note: In Go, the stack doesn't "unwind" on panic. Instead, a panic will traverse up