From 899e46bf433070976f61cb9aa481bb3d7491d80f Mon Sep 17 00:00:00 2001
From: gageorgiev <gageorgiev@vmware.com>
Date: Wed, 11 Aug 2021 11:45:55 +0300
Subject: [PATCH] control-service: Make fluentdconfig configurable

This change allows the fluentdconfig to be altered
through the Values.yml file. The purpose of this is
that making changes to a deployed version of
control-service no long requires pushing changes to
the main control-service repository. The fluentd
block in values.yml includes three values relevant to
the config - filter, match and extra. Filter and match
are inserted into the corresponding filter and match
blocks in the fluentdconfig, whereas extra is any
additional configuration you might include using the
remaining directives.

Testing done:
With the following config in Values.yml:
```yml
fluentd:
  enabled: true
  filter: |-
    @type parser
    key_name log
    reserve_data true
    remove_key_name_field true
    <parse>
      @type json
    </parse>
  match: |-
    @type elastic
```

We get the following config:
```yml
apiVersion: logs.vdp.vmware.com/v1beta1
kind: FluentdConfig
metadata:
  name: pipelines-control-service-parser
spec:
  fluentconf: |
    <filter $labels(app_kubernetes_io/name=pipelines-control-service)>
      @type parser
      key_name log
      reserve_data true
      remove_key_name_field true
      <parse>
        @type json
      </parse>
    </filter>
    <match $labels(app_kubernetes_io/name=pipelines-control-service)>
      @type elastic
    </match>
```

If we extend the config the with a fluentd.extra value:
```yml
  extra: |-
    <source>
      @type http
      port 8090
    </source>
    <system>
      some system config
    </system>
```

We get the following fluentdconfig:
```yml
apiVersion: logs.vdp.vmware.com/v1beta1
kind: FluentdConfig
metadata:
  name: pipelines-control-service-parser
spec:
  fluentconf: |
    <source>
      @type http
      port 8090
    </source>
    <system>
      some system config
    </system>
    <filter $labels(app_kubernetes_io/name=pipelines-control-service)>
      @type parser
      key_name log
      reserve_data true
      remove_key_name_field true
      <parse>
        @type json
      </parse>
    </filter>
    <match $labels(app_kubernetes_io/name=pipelines-control-service)>
      @type elastic
    </match>
```

Signed-off-by: gageorgiev <gageorgiev@vmware.com>
---
 .../templates/fluentdconfig.yaml              | 16 ++++++------
 .../pipelines-control-service/values.yaml     | 25 +++++++++++++++++--
 2 files changed, 31 insertions(+), 10 deletions(-)

diff --git a/projects/control-service/projects/helm_charts/pipelines-control-service/templates/fluentdconfig.yaml b/projects/control-service/projects/helm_charts/pipelines-control-service/templates/fluentdconfig.yaml
index 77dc42f068..bc764e021d 100644
--- a/projects/control-service/projects/helm_charts/pipelines-control-service/templates/fluentdconfig.yaml
+++ b/projects/control-service/projects/helm_charts/pipelines-control-service/templates/fluentdconfig.yaml
@@ -4,16 +4,16 @@ kind: FluentdConfig
 metadata:
   name: {{ include "pipelines-control-service.fullname" . }}-parser
 spec:
-  fluentconf: | # TODO: make fluentd config configurable from Values.yml
+  fluentconf: |
+    {{- if .Values.fluentd.extra }}
+    {{ .Values.fluentd.extra | indent 4 | trim }}
+    {{- end }}
+    {{- if .Values.fluentd.filter }}
     <filter $labels(app_kubernetes_io/name={{ include "pipelines-control-service.name" . }})>
-      @type parser
-      key_name log
-      reserve_data true
-        <parse>
-          @type ltsv
-        </parse>
+      {{ .Values.fluentd.filter | indent 6 | trim }}
     </filter>
+    {{- end }}
     <match $labels(app_kubernetes_io/name={{ include "pipelines-control-service.name" . }})>
-      @type elastic-vrli
+      {{ .Values.fluentd.match | indent 6 | trim }}
     </match>
 {{- end }}
diff --git a/projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml b/projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml
index b095b0a3b2..91aca7c946 100644
--- a/projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml
+++ b/projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml
@@ -676,8 +676,29 @@ datajobTemplate:
               restartPolicy: OnFailure
           ttlSecondsAfterFinished: 600
 
-# TODO: make fluentd config configurable by setting the exact config here and using this variable
 # configuration for the fluentd data collector (https://github.com/vmware/kube-fluentd-operator),
-# which is currently used for collecting and visualizing Versatile Data Kit logs in Kibana
+# which is currently used for collecting and visualizing Versatile Data Kit logs in Kibana;
+# filter and match refer to the fluentd config directives, see here https://docs.fluentd.org/configuration/config-file;
+# extra refers to any additional directives you might use besides for filter and match - note that you must include
+# the directive opening and closing statement in the value of fluentd.extra
+# extra is included at the start of the fluentd config before the filter and match blocks
 fluentd:
   enabled: false
+  filter: |-
+    @type parser
+    key_name log
+    reserve_data true
+    remove_key_name_field true
+    <parse>
+      @type json
+    </parse>
+  match: |-
+    @type elastic
+#  extra: |-
+#    <source>
+#      @type http
+#      port 8090
+#    </source>
+#    <system>
+#      some system config
+#    </system>