[KYUUBI #6864][FOLLOWUP] Support to apply instance with existing labels

### Why are the changes needed?

For histogram and timer metrics, it already has label, so need to support apply the instance label with existing ones.

For example:
```
# HELP kyuubi_backend_service_close_operation Generated from Dropwizard metric import (metric=kyuubi.backend_service.close_operation, type=com.codahale.metrics.Timer)
# TYPE kyuubi_backend_service_close_operation summary
kyuubi_backend_service_close_operation{quantile="0.5",}{instance="hadoopkyuubi-1.hadoopkyuubihl.hadoopmaster-dev.svc.140.tess.io:10019"} 0.032923216000000005
```

### How was this patch tested?

UT.
### Was this patch authored or co-authored using generative AI tooling?

No.

Closes #6868 from turboFei/instance_label_follow.

Closes #6864

4894784 [Wang, Fei] ut
b8f227f [Wang, Fei] save

Authored-by: Wang, Fei <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>

kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/PrometheusReporterService.scala

@@ -140,20 +140,33 @@ class PrometheusReporterService(registry: MetricRegistry)
     val metricsSnapshotWriter = new java.io.StringWriter
     val contentType = TextFormat.chooseContentType(null)
     TextFormat.writeFormat(contentType, metricsSnapshotWriter, bridgeRegistry.metricFamilySamples())
-    val labelStr = labelString(labels)
+    val labelStr = labels.map { case (k, v) => s"""$k="$v"""" }.toArray.sorted.mkString(",")
     metricsSnapshotWriter.toString.split("\n").map { line =>
       if (line.startsWith("#")) {
         line
       } else {
         line.split("\\s+", 2) match {
-          case Array(metrics, rest) => s"""$metrics${labelStr} $rest"""
+          case Array(metrics, rest) =>
+            val metricsWithNewLabels = PrometheusReporterService.applyExtraLabels(metrics, labelStr)
+            s"""$metricsWithNewLabels $rest"""
           case _ => line
         }
       }
     }.mkString("\n")
   }
+}
 
-  private def labelString(labels: Map[String, String]): String = {
-    labels.map { case (k, v) => s"""$k="$v"""" }.toArray.sorted.mkString("{", ",", "}")
+object PrometheusReporterService {
+  def applyExtraLabels(metrics: String, labelStr: String): String = {
+    val labelStartIdx = metrics.indexOf("{")
+    val labelEndIdx = metrics.indexOf("}")
+    if (labelStartIdx > 0 && labelEndIdx > 0) {
+      metrics.substring(0, labelEndIdx).stripSuffix(",") +
+        "," +
+        labelStr +
+        metrics.substring(labelEndIdx)
+    } else {
+      s"$metrics{${labelStr}}"
+    }
   }
 }

kyuubi-metrics/src/test/scala/org/apache/kyuubi/metrics/PrometheusReporterServiceSuite.scala

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.metrics
+
+import org.apache.kyuubi.KyuubiFunSuite
+
+class PrometheusReporterServiceSuite extends KyuubiFunSuite {
+  test("apply extra labels on metrics") {
+    val labels = Map("instance" -> "kyuubi-0")
+    val labelStr = labels.map { case (k, v) => s"""$k="$v"""" }.toArray.sorted.mkString(",")
+    val metrics1 = "kyuubi_backend_service_close_operation{quantile=\"0.5\",}"
+    assert(PrometheusReporterService.applyExtraLabels(metrics1, labelStr) ==
+      "kyuubi_backend_service_close_operation{quantile=\"0.5\",instance=\"kyuubi-0\"}")
+    val metrics2 = "kyuubi_backend_service_close_operation"
+    assert(PrometheusReporterService.applyExtraLabels(metrics2, labelStr) ==
+      "kyuubi_backend_service_close_operation{instance=\"kyuubi-0\"}")
+  }
+}