chore: Add unfinished_work_seconds metric (#1809)

kubernetes-sigs · Nov 14, 2024 · c194a9a · c194a9a
1 parent 3e0c51a
commit c194a9a
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 1 deletion.
diff --git a/pkg/controllers/provisioning/scheduling/metrics.go b/pkg/controllers/provisioning/scheduling/metrics.go
@@ -57,6 +57,19 @@ var (
 			schedulingIDLabel,
 		},
 	)
+	UnfinishedWorkSeconds = opmetrics.NewPrometheusGauge(
+		crmetrics.Registry,
+		prometheus.GaugeOpts{
+			Namespace: metrics.Namespace,
+			Subsystem: schedulerSubsystem,
+			Name:      "unfinished_work_seconds",
+			Help:      "How many seconds of work has been done that is in progress and hasn't been observed by scheduling_duration_seconds.",
+		},
+		[]string{
+			ControllerLabel,
+			schedulingIDLabel,
+		},
+	)
 	IgnoredPodCount = opmetrics.NewPrometheusGauge(
 		crmetrics.Registry,
 		prometheus.GaugeOpts{

diff --git a/pkg/controllers/provisioning/scheduling/scheduler.go b/pkg/controllers/provisioning/scheduling/scheduler.go
@@ -213,6 +213,7 @@ func (s *Scheduler) Solve(ctx context.Context, pods []*corev1.Pod) Results {
 	lastLogTime := s.clock.Now()
 	batchSize := len(q.pods)
 	for {
+		UnfinishedWorkSeconds.Set(float64(s.clock.Since(startTime)), map[string]string{ControllerLabel: injection.GetControllerName(ctx), schedulingIDLabel: string(s.id)})
 		QueueDepth.Set(float64(len(q.pods)), map[string]string{ControllerLabel: injection.GetControllerName(ctx), schedulingIDLabel: string(s.id)})
 
 		if s.clock.Since(lastLogTime) > time.Minute {
@@ -240,7 +241,7 @@ func (s *Scheduler) Solve(ctx context.Context, pods []*corev1.Pod) Results {
 			}
 		}
 	}
-
+	UnfinishedWorkSeconds.Set(0, map[string]string{ControllerLabel: injection.GetControllerName(ctx), schedulingIDLabel: string(s.id)})
 	for _, m := range s.newNodeClaims {
 		m.FinalizeScheduling()
 	}