fix: pod logs label keep (#1177)

* Moved labelsToKeep to end of loki.process

* Updated Meta-Monitoring Example

* Updated Test

charts/k8s-monitoring/charts/feature-pod-logs/templates/_common_log_processing.alloy.tpl

@@ -50,13 +50,6 @@ loki.process "pod_logs" {
     }
   }
 {{- end }}
-{{- with .Values.labelsToKeep }}
-
-  // Only keep the labels that are defined in the `keepLabels` list.
-  stage.label_keep {
-    values = {{ append . "integration" | toJson }}
-  }
-{{- end }}
 
 {{- if or .Values.staticLabels .Values.staticLabelsFrom }}
 
@@ -75,6 +68,14 @@ loki.process "pod_logs" {
 {{ tpl .Values.extraLogProcessingStages $ | indent 2 }}
 {{- end }}
 
+{{- with .Values.labelsToKeep }}
+
+  // Only keep the labels that are defined in the `keepLabels` list.
+  stage.label_keep {
+    values = {{ append . "integration" | toJson }}
+  }
+{{- end }}
+
   forward_to = argument.logs_destinations.value
 }
 

charts/k8s-monitoring/charts/feature-pod-logs/tests/default_test.yaml

@@ -473,7 +473,7 @@ tests:
               argument "logs_destinations" {
                 comment = "Must be a list of log destinations where collected logs should be forwarded to"
               }
-            
+
               discovery.relabel "filtered_pods" {
                 targets = discovery.kubernetes.pods.targets
                 rule {
@@ -498,7 +498,7 @@ tests:
                   replacement = "$1"
                   target_label = "job"
                 }
-            
+
                 // set the container runtime as a label
                 rule {
                   action = "replace"
@@ -507,40 +507,40 @@ tests:
                   replacement = "$1"
                   target_label = "tmp_container_runtime"
                 }
-            
+
                 // set the job label from the k8s.grafana.com/logs.job annotation if it exists
                 rule {
                   source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_logs_job"]
                   regex = "(.+)"
                   target_label = "job"
                 }
-            
+
                 // make all labels on the pod available to the pipeline as labels,
                 // they are omitted before write to loki via stage.label_keep unless explicitly set
                 rule {
                   action = "labelmap"
                   regex = "__meta_kubernetes_pod_label_(.+)"
                 }
-            
+
                 // make all annotations on the pod available to the pipeline as labels,
                 // they are omitted before write to loki via stage.label_keep unless explicitly set
                 rule {
                   action = "labelmap"
                   regex = "__meta_kubernetes_pod_annotation_(.+)"
                 }
               }
-            
+
               discovery.kubernetes "pods" {
                 role = "pod"
                 selectors {
                   role = "pod"
                   field = "spec.nodeName=" + sys.env("HOSTNAME")
                 }
               }
-            
+
               discovery.relabel "filtered_pods_with_paths" {
                 targets = discovery.relabel.filtered_pods.output
-            
+
                 rule {
                   source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"]
                   separator = "/"
@@ -549,22 +549,22 @@ tests:
                   target_label = "__path__"
                 }
               }
-            
+
               local.file_match "pod_logs" {
                 path_targets = discovery.relabel.filtered_pods_with_paths.output
               }
-            
+
               loki.source.file "pod_logs" {
                 targets    = local.file_match.pod_logs.targets
                 forward_to = [loki.process.pod_logs.receiver]
               }
-            
+
               loki.process "pod_logs" {
                 stage.match {
                   selector = "{tmp_container_runtime=~\"containerd|cri-o\"}"
                   // the cri processing stage extracts the following k/v pairs: log, stream, time, flags
                   stage.cri {}
-            
+
                   // Set the extract flags and stream values as labels
                   stage.labels {
                     values = {
@@ -573,20 +573,20 @@ tests:
                     }
                   }
                 }
-            
+
                 stage.match {
                   selector = "{tmp_container_runtime=\"docker\"}"
                   // the docker processing stage extracts the following k/v pairs: log, stream, time
                   stage.docker {}
-            
+
                   // Set the extract stream value as a label
                   stage.labels {
                     values = {
                       stream  = "",
                     }
                   }
                 }
-            
+
                 // Drop the filename label, since it's not really useful in the context of Kubernetes, where we already have cluster,
                 // namespace, pod, and container labels. Drop any structured metadata. Also drop the temporary
                 // container runtime label as it is no longer needed.
@@ -596,19 +596,19 @@ tests:
                     "tmp_container_runtime",
                   ]
                 }
-            
-                // Only keep the labels that are defined in the `keepLabels` list.
-                stage.label_keep {
-                  values = ["app_kubernetes_io_name","container","instance","job","level","namespace","pod","service_name","integration"]
-                }
-            
+
                 stage.static_labels {
                   values = {
                     region = "central",
                     color = sys.env("COLOR"),
                   }
                 }
-            
+
+                // Only keep the labels that are defined in the `keepLabels` list.
+                stage.label_keep {
+                  values = ["app_kubernetes_io_name","container","instance","job","level","namespace","pod","service_name","integration"]
+                }
+
                 forward_to = argument.logs_destinations.value
               }
             }