diff --git a/charts/data-prepper/CHANGELOG.md b/charts/data-prepper/CHANGELOG.md index 4c5888b4..c3f15673 100644 --- a/charts/data-prepper/CHANGELOG.md +++ b/charts/data-prepper/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.3.0] +### Added +- Demo pipeline not part of values.yaml. New value `pipelineConfig.demoPipeline` controls creation of demo pipeline. + ## [0.2.0] ### Added - Added configurable `global.dockerRegistry` diff --git a/charts/data-prepper/Chart.yaml b/charts/data-prepper/Chart.yaml index dfeeea98..dce6739f 100644 --- a/charts/data-prepper/Chart.yaml +++ b/charts/data-prepper/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.2.0 +version: 0.3.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/data-prepper/README.md b/charts/data-prepper/README.md index 40c87464..d06d8ea4 100644 --- a/charts/data-prepper/README.md +++ b/charts/data-prepper/README.md @@ -33,12 +33,12 @@ helm repo update helm install my-data-prepper-release opensearch/data-prepper ``` -Replace my-data-prepper-release with your desired release name. +Replace my-data-prepper-release with your desired release name. When no explicit pipeline is defined, this will configure a demo pipeline using a random source and stdout sink. ## Configuration The Data Prepper Helm chart comes with a variety of configuration options to tailor the deployment to your needs. -The default values are specified in the [values.yaml](values.yaml) file. You can override these values by providing your own values.yaml file during installation or by specifying configuration options with --set flags. +The default values are specified in the [values.yaml](values.yaml) file. You can override these values by providing your own `values.yaml` file during installation or by specifying configuration options with --set flags. For a detailed list of configuration options, refer to the values.yaml file or the [Data Prepper documentation](https://opensearch.org/docs/latest/data-prepper/managing-data-prepper/configuring-data-prepper/). @@ -99,8 +99,11 @@ We welcome contributions! Please read our [CONTRIBUTING.md](../../CONTRIBUTING.m | ingress.tls | list | `[]` | | | nameOverride | string | `""` | Override the default name for the deployment | | nodeSelector | object | `{}` | | -| pipelineConfig | object | `{"config":{"simple-sample-pipeline":{"buffer":{"bounded_blocking":{"batch_size":256,"buffer_size":1024}},"delay":5000,"processor":[{"string_converter":{"upper_case":true}}],"sink":[{"stdout":null}],"source":{"random":null},"workers":2}},"enabled":true,"existingSecret":""}` | Pipeline configuration | -| pipelineConfig.existingSecret | string | `""` | The name of the existing secret containing the pipeline configuration. If enabled is false existingSecret is used. The existingSecret must have a key named `pipelines.yaml`. | +| pipelineConfig | object | (See below) | Pipeline configuration | +| pipelineConfig.enabled | boolean | `false` | Enable inline configuration in `config` sub key. | +| pipelineConfig.config | object | `{}` | Pipeline configuration file inline if `enabled` is set to true | +| pipelineConfig.demoPipeline | boolean | "" | If set, a demo pipeline will be provisioned with source `random` and sink `stdout`. | +| pipelineConfig.existingSecret | string | `""` | The name of an existing secret containing the pipeline configuration. If enabled is false existingSecret is used. The existingSecret must have a key named `pipelines.yaml`. | | podAnnotations | object | `{}` | | | podLabels | object | `{}` | | | podSecurityContext | object | `{}` | | diff --git a/charts/data-prepper/templates/NOTES.txt b/charts/data-prepper/templates/NOTES.txt index cae9ebaf..6814d818 100644 --- a/charts/data-prepper/templates/NOTES.txt +++ b/charts/data-prepper/templates/NOTES.txt @@ -20,3 +20,11 @@ echo "Visit http://127.0.0.1:8080 to use your application" kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT {{- end }} + +{{- if eq "true" (include "data-prepper.demoPipeline" .) }} +The demo pipeline configuration is enabled, using the `random` source and `stdout` sink. +{{- else if .Values.pipelineConfig.enabled }} +Inline pipeline configuration is enabled. Please refer to the values.yaml file for the configuration. +{{- else }} +Pipeline configuration from secret {{ .Values.pipelineConfig.existingSecret }} is enabled. +{{- end }} \ No newline at end of file diff --git a/charts/data-prepper/templates/_helpers.tpl b/charts/data-prepper/templates/_helpers.tpl index 4f808c76..e10f00a2 100644 --- a/charts/data-prepper/templates/_helpers.tpl +++ b/charts/data-prepper/templates/_helpers.tpl @@ -71,3 +71,25 @@ Create the dockerRegistry prefix if defined {{- .Values.global.dockerRegistry | trimSuffix "/" | printf "%s/" -}} {{- end -}} {{- end -}} + +{{/* +Utility to decide whether demoPipeline should be enabled. +If pipelineConfig.demoPipeline is true/false, return that value. +If pipelineConfig.demoPipeline is undefined, return true if no other pipeline is configured. +*/}} +{{- define "data-prepper.demoPipeline" -}} +{{- $demoPipeline := .Values.pipelineConfig.demoPipeline | toString -}} +{{- if eq $demoPipeline "true" -}} +{{ true }} +{{- else if eq $demoPipeline "false" -}} +{{ false }} +{{- else if or (not $demoPipeline) (eq $demoPipeline "") -}} +{{- if and (not .Values.pipelineConfig.enabled) (not .Values.pipelineConfig.existingSecret) -}} +{{ true }} +{{- else -}} +{{ false }} +{{- end -}} +{{- else -}} +{{ false }} +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/data-prepper/templates/demoPipeline.yaml b/charts/data-prepper/templates/demoPipeline.yaml new file mode 100644 index 00000000..5c22c985 --- /dev/null +++ b/charts/data-prepper/templates/demoPipeline.yaml @@ -0,0 +1,25 @@ +{{- if eq "true" (include "data-prepper.demoPipeline" .) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "data-prepper.fullname" . }}-demo-pipeline + labels: + {{- include "data-prepper.labels" . | nindent 4 }} +type: Opaque +stringData: + pipelines.yaml: | + simple-sample-pipeline: + workers: 2 # the number of workers + delay: 5000 # in milliseconds, how long workers wait between read attempts + source: + random: {} + buffer: + bounded_blocking: + buffer_size: 1024 # max number of records the buffer accepts + batch_size: 256 # max number of records the buffer drains after each read + processor: + - string_converter: + upper_case: true + sink: + - stdout: {} +{{- end }} diff --git a/charts/data-prepper/templates/deployment.yaml b/charts/data-prepper/templates/deployment.yaml index 61327d85..25671b80 100644 --- a/charts/data-prepper/templates/deployment.yaml +++ b/charts/data-prepper/templates/deployment.yaml @@ -90,10 +90,12 @@ spec: name: {{ include "data-prepper.fullname" . }}-config - name: data-prepper-pipelines secret: - {{- if .Values.pipelineConfig.enabled }} + {{- if eq "true" (include "data-prepper.demoPipeline" .) }} + secretName: {{ include "data-prepper.fullname" . }}-demo-pipeline + {{- else if .Values.pipelineConfig.enabled }} secretName: {{ include "data-prepper.fullname" . }}-pipeline {{- else }} - secretName: {{ required "A valid .Values.pipelineConfig.existingSecret entry required!" .Values.pipelineConfig.existingSecret }} + secretName: {{ required "A valid pipeline configuration is required! Please set one of:\npipelineConfig.existingSecret - for your custom pipeline secret\npipelineConfig.enabled - for inline configuration in values.yaml or\npipelineConfig.demoPipeline=true - for a sample demo pipeline using random source and stdout sink." .Values.pipelineConfig.existingSecret }} {{- end }} {{- with .Values.volumes }} {{- toYaml . | nindent 8 }} diff --git a/charts/data-prepper/values.yaml b/charts/data-prepper/values.yaml index d74efa25..02f80f94 100644 --- a/charts/data-prepper/values.yaml +++ b/charts/data-prepper/values.yaml @@ -101,160 +101,31 @@ config: # -- Pipeline configuration pipelineConfig: - # If enabled, a secret containing the pipeline configuration will be created based on the 'config' section below. - enabled: true + # If 'true', a secret containing a demo pipeline configuration with random source and stdout sink will be created. + # If left undefined, the demo pipeline will be used only when no other pipeline is configured below + demoPipeline: "" # -- The name of the existing secret containing the pipeline configuration. # If enabled is false existingSecret is used. The existingSecret must have a key named `pipelines.yaml`. existingSecret: "" + # If enabled, a secret containing the pipeline configuration will be created based on the 'config' section below. + enabled: false # The configuration of the pipeline see https://opensearch.org/docs/2.4/data-prepper/pipelines/pipelines/ config: - ## Simple Example - simple-sample-pipeline: - workers: 2 # the number of workers - delay: 5000 # in milliseconds, how long workers wait between read attempts - source: - random: - buffer: - bounded_blocking: - buffer_size: 1024 # max number of records the buffer accepts - batch_size: 256 # max number of records the buffer drains after each read - processor: - - string_converter: - upper_case: true - sink: - - stdout: - - ## More Complex example - # otel-logs-pipeline: - # workers: 5 - # delay: 10 - # source: - # otel_logs_source: - # ssl: false - # buffer: - # bounded_blocking: - # sink: - # - opensearch: - # hosts: ["https://opensearch-cluster-master:9200"] - # username: "admin" - # password: "admin" - # insecure: true - # index_type: custom - # index: events-%{yyyy.MM.dd} - # #max_retries: 20 - # bulk_size: 4 - # otel-trace-pipeline: - # # workers is the number of threads processing data in each pipeline. - # # We recommend same value for all pipelines. - # # default value is 1, set a value based on the machine you are running Data Prepper - # workers: 8 - # # delay in milliseconds is how often the worker threads should process data. - # # Recommend not to change this config as we want the otel-trace-pipeline to process as quick as possible - # # default value is 3_000 ms - # delay: "100" - # source: - # otel_trace_source: - # ssl: false # Change this to enable encryption in transit - # buffer: - # bounded_blocking: - # # buffer_size is the number of ExportTraceRequest from otel-collector the data prepper should hold in memeory. - # # We recommend to keep the same buffer_size for all pipelines. - # # Make sure you configure sufficient heap - # # default value is 12800 - # buffer_size: 25600 - # # This is the maximum number of request each worker thread will process within the delay. - # # Default is 200. - # # Make sure buffer_size >= workers * batch_size - # batch_size: 400 - # sink: - # - pipeline: - # name: "raw-traces-pipeline" - # - pipeline: - # name: "otel-service-map-pipeline" - # raw-traces-pipeline: - # workers: 5 - # delay: 3000 - # source: - # pipeline: - # name: "otel-trace-pipeline" - # buffer: - # bounded_blocking: - # buffer_size: 25600 # max number of records the buffer accepts - # batch_size: 400 # max number of records the buffer drains after each read - # processor: - # - otel_traces: - # - otel_trace_group: - # hosts: [ "https://opensearch-cluster-master:9200" ] - # insecure: true - # username: "admin" - # password: "admin" - # sink: - # - opensearch: - # hosts: ["https://opensearch-cluster-master:9200"] - # username: "admin" - # password: "admin" - # insecure: true - # index_type: trace-analytics-raw - # otel-service-map-pipeline: - # workers: 5 - # delay: 3000 - # source: - # pipeline: - # name: "otel-trace-pipeline" - # processor: - # - service_map: - # # The window duration is the maximum length of time the data prepper stores the most recent trace data to evaluvate service-map relationships. - # # The default is 3 minutes, this means we can detect relationships between services from spans reported in last 3 minutes. - # # Set higher value if your applications have higher latency. - # window_duration: 180 - # buffer: - # bounded_blocking: - # # buffer_size is the number of ExportTraceRequest from otel-collector the data prepper should hold in memeory. - # # We recommend to keep the same buffer_size for all pipelines. - # # Make sure you configure sufficient heap - # # default value is 12800 - # buffer_size: 25600 - # # This is the maximum number of request each worker thread will process within the delay. - # # Default is 200. - # # Make sure buffer_size >= workers * batch_size - # batch_size: 400 - # sink: - # - opensearch: - # hosts: ["https://opensearch-cluster-master:9200"] - # username: "admin" - # password: "admin" - # insecure: true - # index_type: trace-analytics-service-map - # #index: otel-v1-apm-span-%{yyyy.MM.dd} - # #max_retries: 20 - # bulk_size: 4 - # otel-metrics-pipeline: - # workers: 8 - # delay: 3000 + ## Provide your pipeline configuration here if 'enabled' is set to true. See documentation for more advanced pipelines + # simple-sample-pipeline: + # workers: 2 # the number of workers + # delay: 5000 # in milliseconds, how long workers wait between read attempts # source: - # otel_metrics_source: - # health_check_service: true - # ssl: false + # random: {} # buffer: # bounded_blocking: - # buffer_size: 1024 # max number of records the buffer accepts - # batch_size: 1024 # max number of records the buffer drains after each read + # buffer_size: 1024 # max number of records the buffer accepts + # batch_size: 256 # max number of records the buffer drains after each read # processor: - # - otel_metrics: - # calculate_histogram_buckets: true - # calculate_exponential_histogram_buckets: true - # exponential_histogram_max_allowed_scale: 10 - # flatten_attributes: false + # - string_converter: + # upper_case: true # sink: - # - opensearch: - # hosts: ["https://opensearch-cluster-master:9200"] - # username: "admin" - # password: "admin" - # insecure: true - # index_type: custom - # index: metrics-%{yyyy.MM.dd} - # #max_retries: 20 - # bulk_size: 4 + # - stdout: {} # -- Data Prepper ports ports: