exporter: add alert for storagequotautilizationration > 0.8

metrics/deploy/prometheus-ocs-rules.yaml

@@ -354,6 +354,16 @@ spec:
         floor((ocs_storage_provider_operator_version>0)/1000) - ignoring(storage_consumer_name) group_right() floor((ocs_storage_client_operator_version>0)/1000) > 1 or floor((ocs_storage_client_operator_version>0)/1000) - ignoring(storage_consumer_name) group_left() floor((ocs_storage_provider_operator_version>0)/1000) >= 1
       labels:
         severity: critical
+    - alert: StorageQuotaUtilizationThresholdReached
+      annotations:
+        description: ODF client ({{ $labels.client_name }}) - ({{ $labels.client_cluster_name }}) storage consumption has surpassed 80% of the configured quota, this might result in issues in the creation of new PV/PVCs on the target cluster.
+        message: ODF client ({{ $labels.odf_client_name }}) storage consumption has surpassed 80% of the configured quota, this might result in issues in the creation of new PV/PVCs on the target cluster.
+        runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/openshift-container-storage-operator/StorageQuotaUtilizationThresholdReached.md
+        severity_level: warning
+      expr: |
+        ocs_storage_client_storage_quota_utilization_ratio > 0.8
+      labels:
+        severity: warning
   - name: storage-cluster-alerts.rules
     rules:
     - alert: CephMonLowNumber

metrics/internal/collectors/storageconsumer.go

@@ -19,12 +19,13 @@ import (
 var _ prometheus.Collector = &StorageConsumerCollector{}
 
 type StorageConsumerCollector struct {
-	Informer                cache.SharedIndexInformer
-	StorageConsumerMetadata *prometheus.Desc
-	LastHeartbeat           *prometheus.Desc
-	ProviderOperatorVersion *prometheus.Desc
-	ClientOperatorVersion   *prometheus.Desc
-	AllowedNamespace        string
+	Informer                     cache.SharedIndexInformer
+	StorageConsumerMetadata      *prometheus.Desc
+	LastHeartbeat                *prometheus.Desc
+	StorageQuotaUtilizationRatio *prometheus.Desc
+	ProviderOperatorVersion      *prometheus.Desc
+	ClientOperatorVersion        *prometheus.Desc
+	AllowedNamespace             string
 }
 
 func NewStorageConsumerCollector(opts *options.Options) *StorageConsumerCollector {
@@ -60,6 +61,12 @@ func NewStorageConsumerCollector(opts *options.Options) *StorageConsumerCollecto
 			[]string{"storage_consumer_name"},
 			nil,
 		),
+		StorageQuotaUtilizationRatio: prometheus.NewDesc(
+			prometheus.BuildFQName("ocs", "storage_client", "storage_quota_utilization_ratio"),
+			`StorageQuotaUtilizationRatio of ODF Storage Client`,
+			[]string{"client_name", "client_cluster_name"},
+			nil,
+		),
 		Informer: sharedIndexInformer,
 	}
 }
@@ -78,6 +85,7 @@ func (c *StorageConsumerCollector) Describe(ch chan<- *prometheus.Desc) {
 		c.LastHeartbeat,
 		c.ProviderOperatorVersion,
 		c.ClientOperatorVersion,
+		c.StorageQuotaUtilizationRatio,
 	}
 	for _, d := range ds {
 		ch <- d
@@ -137,6 +145,10 @@ func (c *StorageConsumerCollector) collectStorageConsumersMetadata(storageConsum
 			prometheus.GaugeValue,
 			float64(encodeVersion(storageConsumer.Status.Client.OperatorVersion)),
 			storageConsumer.Name)
+
+		ch <- prometheus.MustNewConstMetric(c.StorageQuotaUtilizationRatio,
+			prometheus.GaugeValue, storageConsumer.Status.Client.StorageQuotaUtilizationRatio,
+			storageConsumer.Status.Client.Name, storageConsumer.Status.Client.ClusterName)
 	}
 }