Merge branch 'main' of https://github.com/STCLab-Inc/wave-autoscale i…

…nto 181-metric-integrations-from-telegraf

core/data-layer/src/types/scaling_component_definition.rs

@@ -8,6 +8,9 @@ use ts_rs::TS;
 fn default_kind() -> ObjectKind {
     ObjectKind::ScalingComponent
 }
+fn default_metadata() -> HashMap<String, Value> {
+    HashMap::new()
+}
 
 #[derive(TS)]
 #[ts(
@@ -25,5 +28,6 @@ pub struct ScalingComponentDefinition {
     pub id: String,
     pub component_kind: String,
     #[ts(type = "object")]
+    #[serde(default = "default_metadata")]
     pub metadata: HashMap<String, Value>,
 }

examples/metrics/vector/postgresql.yaml

@@ -0,0 +1,31 @@
+# We obtain the Istio request duration metric value from Prometheus through the vector agent.
+kind: Metric
+id: postgresql
+collector: vector
+metadata:
+  sources:
+    my_source_id_1:
+      type: postgresql_metrics
+      # If you use the Prometheus endpoint at `/api/v1/query`, you will receive a value in JSON format.
+      # Therefore, you need to use Vector's transforms to convert it to match the format of wave-autoscale.
+      endpoints:
+        - postgres://user:password@url/postgres
+      exclude_databases:
+        - ^postgres$
+      include_databases:
+        - ^postgres$
+      namespace: postgresql
+      scrape_interval_secs: 30
+  sinks:
+    my_sinks_id:
+      # The 'sinks' type in Vector should be set to 'wave-autoscale'.
+      type: wave-autoscale
+      # For the 'inputs' value, you should basically provide the 'id' value from 'sources'.
+      # If there are 'transforms', you can input the 'id' of the last transformed form.
+      inputs: ["my_source_id_1"]
+########### [ Saved Metric Data Example ] ############
+# pg_stat_bgwriter_maxwritten_clean_total
+# {
+#   [{"name": "up","value": 1.0,"timestamp": "2023-10-31T08:57:56.126699Z"},{"name": "pg_stat_bgwriter_checkpoints_timed_total","value": 1783.0,"timestamp": "2023-10-31T08:57:56.124242Z"},{"name": "pg_stat_bgwriter_checkpoints_req_total","value": 10.0,"timestamp": "2023-10-31T08:57:56.124253Z"},{"name": "pg_stat_bgwriter_checkpoint_write_time_seconds_total","value": 322.164,"timestamp": "2023-10-31T08:57:56.124261Z"},{"name": "pg_stat_bgwriter_checkpoint_sync_time_seconds_total","value": 5.966,"timestamp": "2023-10-31T08:57:56.124264Z"},{"name": "pg_stat_bgwriter_buffers_checkpoint_total","value": 4136.0,"timestamp": "2023-10-31T08:57:56.124268Z"},{"name": "pg_stat_bgwriter_buffers_clean_total","value": 0.0,"timestamp": "2023-10-31T08:57:56.124278Z"},{"name": "pg_stat_bgwriter_maxwritten_clean_total","value": 0.0,"timestamp": "2023-10-31T08:57:56.124282Z"},{"name": "pg_stat_bgwriter_buffers_backend_total","value": 458.0,"timestamp": "2023-10-31T08:57:56.124285Z"},{"name": "pg_stat_bgwriter_buffers_backend_fsync_total","value": 0.0,"timestamp": "2023-10-31T08:57:56.124291Z"},{"name": "pg_stat_bgwriter_buffers_alloc_total","value": 9460.0,"timestamp": "2023-10-31T08:57:56.124296Z"},{"name": "pg_stat_bgwriter_stats_reset","value": 1687908327.0,"timestamp": "2023-10-31T08:57:56.124627Z"}]
+# }
+######################################################

examples/metrics/vector/prometheus_coredns_request.yaml

@@ -0,0 +1,46 @@
+# We obtain the Istio request duration metric value from Prometheus through the vector agent.
+kind: Metric
+id: prometheus_coredns_request_5m
+collector: vector
+metadata:
+  sources:
+    my_source_id_1:
+      type: http_client
+      # If you use the Prometheus endpoint at `/api/v1/query`, you will receive a value in JSON format.
+      # Therefore, you need to use Vector's transforms to convert it to match the format of wave-autoscale.
+      endpoint: "http://localhost:9090/api/v1/query"
+      scrape_interval_secs: 30
+      # CoreDNS request rate of increase
+      # This metric is likely to rise as your application sees spikes in activity or during cyclical increases in usage.
+      # type 'A': ipv4, 'AAAA': ipv6
+      query:
+        "query": ['rate(coredns_dns_requests_total{type=~"A|AAAA"}[5m])']
+  transforms:
+    my_transforms_id_1:
+      inputs: ["my_source_id_1"]
+      type: remap
+      # - The metric format for wave-autoscale is as follows:
+      #   {"name": "coredns_request", "tags": {"tag_id": "tag1"}, "timestamp": 1698122911, "gauge": {"value" : 10.1}}
+      # - Both 'tags' and 'timestamp' are optional.
+      # - gauge.value must be a number.
+      # - VRL Playground - https://playground.vrl.dev/
+      # - endpoint metric data example:
+      #   {"status": "success", "data": {"resultType": "vector", "result": [{"metric": {"destination_app": "order-server"}, "value": [1698134195.218, "0"]}, {"metric": {"connection_security_policy": "mutual_tls"}, "value": [1698134195.218, "0"]}, {"metric": {"connection_security_policy": "unknown"}, "value": [1698134195.218, "0"]}]}}
+      source: |-
+        . = parse_json!(.message)
+        tally = []
+        for_each(array!(.data.result)) -> |_index, value| {
+            tally = push(tally, {"name": "coredns_request", "tags": value.metric, "timestamp": value.value[0], "gauge": {"value" : to_float!(value.value[1])}})
+        }
+        tally
+        . = tally
+  sinks:
+    my_sinks_id:
+      # The 'sinks' type in Vector should be set to 'wave-autoscale'.
+      type: wave-autoscale
+      # For the 'inputs' value, you should basically provide the 'id' value from 'sources'.
+      # If there are 'transforms', you can input the 'id' of the last transformed form.
+      inputs: ["my_transforms_id_1"]
+########### [ Saved Metric Data Example ] ############
+# [{"name":"coredns_request","tags":{"container":"coredns","endpoint":"http-metrics","family":"1","instance":"172.31.16.95:9153","job":"coredns","namespace":"kube-system","pod":"coredns-8496bbc677-ssw8p","proto":"udp","server":"dns://:53","service":"prometheus-community-kube-coredns","type":"A","zone":"."},"value":0.5111111111111112,"timestamp":1698983669.199},{"name":"coredns_request","tags":{"container":"coredns","endpoint":"http-metrics","family":"1","instance":"172.31.16.95:9153","job":"coredns","namespace":"kube-system","pod":"coredns-8496bbc677-ssw8p","proto":"udp","server":"dns://:53","service":"prometheus-community-kube-coredns","type":"AAAA","zone":"."},"value":0.003703703703703704,"timestamp":1698983669.199},{"name":"coredns_request","tags":{"container":"coredns","endpoint":"http-metrics","family":"1","instance":"172.31.43.235:9153","job":"coredns","namespace":"kube-system","pod":"coredns-8496bbc677-hqwp8","proto":"udp","server":"dns://:53","service":"prometheus-community-kube-coredns","type":"A","zone":"."},"value":0.45555555555555555,"timestamp":1698983669.199},{"name":"coredns_request","tags":{"container":"coredns","endpoint":"http-metrics","family":"1","instance":"172.31.43.235:9153","job":"coredns","namespace":"kube-system","pod":"coredns-8496bbc677-hqwp8","proto":"udp","server":"dns://:53","service":"prometheus-community-kube-coredns","type":"AAAA","zone":"."},"value":0.007407407407407408,"timestamp":1698983669.199},{"name":"coredns_request","tags":{"container":"coredns","endpoint":"http-metrics","family":"1","instance":"172.31.45.158:9153","job":"coredns","namespace":"kube-system","pod":"coredns-8496bbc677-t2b4k","proto":"udp","server":"dns://:53","service":"prometheus-community-kube-coredns","type":"A","zone":"."},"value":0.7592592592592593,"timestamp":1698983669.199},{"name":"coredns_request","tags":{"container":"coredns","endpoint":"http-metrics","family":"1","instance":"172.31.45.158:9153","job":"coredns","namespace":"kube-system","pod":"coredns-8496bbc677-t2b4k","proto":"udp","server":"dns://:53","service":"prometheus-community-kube-coredns","type":"AAAA","zone":"."},"value":0.011111111111111112,"timestamp":1698983669.199}]
+######################################################

examples/metrics/vector/prometheus_istio_request_duration.yaml

@@ -0,0 +1,48 @@
+# We obtain the Istio request duration metric value from Prometheus through the vector agent.
+kind: Metric
+id: prometheus_istio_request_duration_1m
+collector: vector
+metadata:
+  sources:
+    my_source_id_1:
+      type: http_client
+      # If you use the Prometheus endpoint at `/api/v1/query`, you will receive a value in JSON format.
+      # Therefore, you need to use Vector's transforms to convert it to match the format of wave-autoscale.
+      endpoint: "http://localhost:9090/api/v1/query"
+      scrape_interval_secs: 30
+      query:
+        "query":
+          [
+            'rate(istio_request_duration_milliseconds_sum{response_code="200"}[1m])',
+          ]
+  transforms:
+    my_transforms_id_1:
+      inputs: ["my_source_id_1"]
+      type: remap
+      # - The metric format for wave-autoscale is as follows:
+      #   {"name": "name_1", "tags": {"tag_id": "tag1"}, "timestamp": 1698122911, "gauge": {"value" : 10.1}}
+      # - Both 'tags' and 'timestamp' are optional.
+      # - gauge.value must be a number.
+      # - VRL Playground - https://playground.vrl.dev/
+      # - endpoint metric data example:
+      #   {"status": "success", "data": {"resultType": "vector", "result": [{"metric": {"destination_app": "order-server"}, "value": [1698134195.218, "0"]}, {"metric": {"connection_security_policy": "mutual_tls"}, "value": [1698134195.218, "0"]}, {"metric": {"connection_security_policy": "unknown"}, "value": [1698134195.218, "0"]}]}}
+      source: |-
+        . = parse_json!(.message)
+        tally = []
+        for_each(array!(.data.result)) -> |_index, value| {
+            tally = push(tally, {"name": "name_1", "tags": value.metric, "timestamp": value.value[0], "gauge": {"value" : to_float!(value.value[1])}})
+        }
+        tally
+        . = tally
+  sinks:
+    my_sinks_id:
+      # The 'sinks' type in Vector should be set to 'wave-autoscale'.
+      type: wave-autoscale
+      # For the 'inputs' value, you should basically provide the 'id' value from 'sources'.
+      # If there are 'transforms', you can input the 'id' of the last transformed form.
+      inputs: ["my_transforms_id_1"]
+########### [ Saved Metric Data Example ] ############
+# {
+#   [{"name":"name_1","tags":{"connection_security_policy":"mutual_tls","destination_app":"order-server","destination_canonical_revision":"latest","destination_canonical_service":"order-server","destination_cluster":"Kubernetes","destination_principal":"spiffe://cluster.local/ns/wave-autoscale/sa/default","destination_service":"order-server-sv.wave-autoscale.svc.cluster.local","destination_service_name":"order-server-sv","destination_service_namespace":"wave-autoscale","destination_workload":"order-server-dp","destination_workload_namespace":"wave-autoscale","instance":"172.31.29.162:15090","job":"envoy-stats","reporter":"destination","request_protocol":"http","response_code":"200","response_flags":"-","source_app":"istio-ingressgateway","source_canonical_revision":"latest","source_canonical_service":"istio-ingressgateway","source_cluster":"Kubernetes","source_principal":"spiffe://cluster.local/ns/istio-system/sa/istio-ingressgateway","source_version":"unknown","source_workload":"istio-ingressgateway","source_workload_namespace":"istio-system"},"value":"0","timestamp":1698304631.731},{"name":"name_1","tags":{"connection_security_policy":"mutual_tls","destination_app":"product-server","destination_canonical_revision":"latest","destination_canonical_service":"product-server","destination_cluster":"Kubernetes","destination_principal":"spiffe://cluster.local/ns/wave-autoscale/sa/default","destination_service":"product-server-sv.wave-autoscale.svc.cluster.local","destination_service_name":"product-server-sv","destination_service_namespace":"wave-autoscale","destination_workload":"product-server-dp","destination_workload_namespace":"wave-autoscale","instance":"172.31.39.238:15090","job":"envoy-stats","reporter":"destination","request_protocol":"http","response_code":"200","response_flags":"-","source_app":"istio-ingressgateway","source_canonical_revision":"latest","source_canonical_service":"istio-ingressgateway","source_cluster":"Kubernetes","source_principal":"spiffe://cluster.local/ns/istio-system/sa/istio-ingressgateway","source_version":"unknown","source_workload":"istio-ingressgateway","source_workload_namespace":"istio-system"},"value":"0","timestamp":1698304631.731},{"name":"name_1","tags":{"connection_security_policy":"unknown","destination_app":"order-server","destination_canonical_revision":"latest","destination_canonical_service":"order-server","destination_cluster":"Kubernetes","destination_principal":"spiffe://cluster.local/ns/wave-autoscale/sa/default","destination_service":"order-server-sv.wave-autoscale.svc.cluster.local","destination_service_name":"order-server-sv","destination_service_namespace":"wave-autoscale","destination_version":"unknown","destination_workload":"order-server-dp","destination_workload_namespace":"wave-autoscale","instance":"172.31.34.170:15090","job":"envoy-stats","reporter":"source","request_protocol":"http","response_code":"200","response_flags":"-","source_app":"istio-ingressgateway","source_canonical_revision":"latest","source_canonical_service":"istio-ingressgateway","source_cluster":"Kubernetes","source_principal":"spiffe://cluster.local/ns/istio-system/sa/istio-ingressgateway","source_workload":"istio-ingressgateway","source_workload_namespace":"istio-system"},"value":"0","timestamp":1698304631.731},{"name":"name_1","tags":{"connection_security_policy":"unknown","destination_app":"product-server","destination_canonical_revision":"latest","destination_canonical_service":"product-server","destination_cluster":"Kubernetes","destination_principal":"spiffe://cluster.local/ns/wave-autoscale/sa/default","destination_service":"product-server-sv.wave-autoscale.svc.cluster.local","destination_service_name":"product-server-sv","destination_service_namespace":"wave-autoscale","destination_version":"unknown","destination_workload":"product-server-dp","destination_workload_namespace":"wave-autoscale","instance":"172.31.34.170:15090","job":"envoy-stats","reporter":"source","request_protocol":"http","response_code":"200","response_flags":"-","source_app":"istio-ingressgateway","source_canonical_revision":"latest","source_canonical_service":"istio-ingressgateway","source_cluster":"Kubernetes","source_principal":"spiffe://cluster.local/ns/istio-system/sa/istio-ingressgateway","source_workload":"istio-ingressgateway","source_workload_namespace":"istio-system"},"value":"0","timestamp":1698304631.731},{"name":"name_1","tags":{"connection_security_policy":"unknown","destination_app":"product-server","destination_canonical_revision":"latest","destination_canonical_service":"product-server","destination_cluster":"Kubernetes","destination_principal":"spiffe://cluster.local/ns/wave-autoscale/sa/default","destination_service":"product-server-sv.wave-autoscale.svc.cluster.local","destination_service_name":"product-server-sv","destination_service_namespace":"wave-autoscale","destination_version":"unknown","destination_workload":"product-server-dp","destination_workload_namespace":"wave-autoscale","instance":"172.31.34.170:15090","job":"envoy-stats","reporter":"source","request_protocol":"http","response_code":"200","response_flags":"DI","source_app":"istio-ingressgateway","source_canonical_revision":"latest","source_canonical_service":"istio-ingressgateway","source_cluster":"Kubernetes","source_principal":"spiffe://cluster.local/ns/istio-system/sa/istio-ingressgateway","source_workload":"istio-ingressgateway","source_workload_namespace":"istio-system"},"value":"0","timestamp":1698304631.731},{"name":"name_1","tags":{"connection_security_policy":"unknown","destination_app":"unknown","destination_canonical_revision":"2.47.0","destination_canonical_service":"prometheus","destination_cluster":"Kubernetes","destination_principal":"unknown","destination_service":"prometheus-community-kube-prometheus.istio-system.svc.cluster.local","destination_service_name":"prometheus-community-kube-prometheus","destination_service_namespace":"istio-system","destination_version":"unknown","destination_workload":"prometheus-prometheus-community-kube-prometheus","destination_workload_namespace":"istio-system","instance":"172.31.34.170:15090","job":"envoy-stats","reporter":"source","request_protocol":"http","response_code":"200","response_flags":"-","source_app":"istio-ingressgateway","source_canonical_revision":"latest","source_canonical_service":"istio-ingressgateway","source_cluster":"Kubernetes","source_principal":"unknown","source_workload":"istio-ingressgateway","source_workload_namespace":"istio-system"},"value":"0.1349999999998545","timestamp":1698304631.731},{"name":"name_1","tags":{"connection_security_policy":"unknown","destination_app":"unknown","destination_canonical_revision":"latest","destination_canonical_service":"grafana","destination_cluster":"Kubernetes","destination_principal":"unknown","destination_service":"prometheus-community-grafana.istio-system.svc.cluster.local","destination_service_name":"prometheus-community-grafana","destination_service_namespace":"istio-system","destination_version":"unknown","destination_workload":"prometheus-community-grafana","destination_workload_namespace":"istio-system","instance":"172.31.34.170:15090","job":"envoy-stats","reporter":"source","request_protocol":"http","response_code":"200","response_flags":"-","source_app":"istio-ingressgateway","source_canonical_revision":"latest","source_canonical_service":"istio-ingressgateway","source_cluster":"Kubernetes","source_principal":"unknown","source_workload":"istio-ingressgateway","source_workload_namespace":"istio-system"},"value":"0","timestamp":1698304631.731},{"name":"name_1","tags":{"connection_security_policy":"unknown","destination_app":"unknown","destination_canonical_revision":"v1.74.0","destination_canonical_service":"kiali","destination_cluster":"Kubernetes","destination_principal":"unknown","destination_service":"kiali.istio-system.svc.cluster.local","destination_service_name":"kiali","destination_service_namespace":"istio-system","destination_version":"unknown","destination_workload":"kiali","destination_workload_namespace":"istio-system","instance":"172.31.34.170:15090","job":"envoy-stats","reporter":"source","request_protocol":"http","response_code":"200","response_flags":"-","source_app":"istio-ingressgateway","source_canonical_revision":"latest","source_canonical_service":"istio-ingressgateway","source_cluster":"Kubernetes","source_principal":"unknown","source_workload":"istio-ingressgateway","source_workload_namespace":"istio-system"},"value":"0","timestamp":1698304631.731}]
+# }
+######################################################

examples/metrics/vector/prometheus_nginx_connections_waiting.yaml

@@ -0,0 +1,30 @@
+# We obtain the metric value of connections waiting in Nginx from Prometheus through the vector agent.
+kind: Metric
+id: prometheus_nginx_connections_waiting
+collector: vector
+metadata:
+  sources:
+    my_source_id_1:
+      type: prometheus_scrape
+      # If you use the Prometheus endpoint at `/api/v1/query`, you will receive a value in JSON format.
+      # Therefore, you need to use Vector's transforms to convert it to match the format of wave-autoscale.
+      endpoints:
+        - "http://localhost:9090/federate"
+      scrape_interval_secs: 30
+      query:
+        "match[]":
+          - "nginx_connections_waiting"
+      # endpoint metric data example:
+      # nginx_connections_waiting{endpoint="http://nginx-10.0.101.222:80/nginx_status",host="nginx-10.0.101.222:80",instance="",monitor="prometheus_center"} 0 1698133601723
+  sinks:
+    my_sinks_id:
+      # The 'sinks' type in Vector should be set to 'wave-autoscale'.
+      type: wave-autoscale
+      # For the 'inputs' value, you should basically provide the 'id' value from 'sources'.
+      # If there are 'transforms', you can input the 'id' of the last transformed form.
+      inputs: ["my_source_id_1"]
+########### [ Saved Metric Data Example ] ############
+# {
+#   [{"name":"nginx_connections_waiting","tags":{"endpoint":"http://nginx-10.0.103.99:80/nginx_status","host":"nginx-10.0.103.99:80","instance":"","monitor":"prometheus_center"},"value":0.0,"timestamp":"2023-10-26T07:11:39.470Z"}]
+# }
+######################################################