From ed24c3a1256dabecfc9e6910960654c6c60fc45d Mon Sep 17 00:00:00 2001
From: Austin Parker <austin@ap2.io>
Date: Tue, 3 Oct 2023 15:58:30 -0400
Subject: [PATCH] OpenSearch Spike (#1151)

* add opensearch

* fix env

* fix yamllint

* fix lint

* fix errors in datasource

* remove stdout export
---
 docker-compose.yml                            | 33 +++++++++++++++++++
 .../provisioning/datasources/opensearch.yaml  | 20 +++++++++++
 src/opensearch/data-prepper-config.yaml       |  4 +++
 src/opensearch/pipelines.yaml                 | 14 ++++++++
 src/otelcollector/otelcol-observability.yml   |  6 ++++
 5 files changed, 77 insertions(+)
 create mode 100644 src/grafana/provisioning/datasources/opensearch.yaml
 create mode 100644 src/opensearch/data-prepper-config.yaml
 create mode 100644 src/opensearch/pipelines.yaml

diff --git a/docker-compose.yml b/docker-compose.yml
index ab162f5639..584e403129 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -638,6 +638,8 @@ services:
       resources:
         limits:
           memory: 100M
+    environment:
+      - "GF_INSTALL_PLUGINS=grafana-opensearch-datasource"
     volumes:
       - ./src/grafana/grafana.ini:/etc/grafana/grafana.ini
       - ./src/grafana/provisioning/:/etc/grafana/provisioning/
@@ -692,6 +694,37 @@ services:
       - "${PROMETHEUS_SERVICE_PORT}:${PROMETHEUS_SERVICE_PORT}"
     logging: *logging
 
+  opensearch:
+    image: opensearchproject/opensearch:latest
+    container_name: opensearch
+    environment:
+      - cluster.name=demo-cluster
+      - node.name=demo-node
+      - bootstrap.memory_lock=true
+      - discovery.type=single-node
+      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
+      - "DISABLE_INSTALL_DEMO_CONFIG=true"
+      - "DISABLE_SECURITY_PLUGIN=true"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    ports:
+      - "9200:9200"
+    logging: *logging
+
+  dataprepper:
+    image: opensearchproject/data-prepper:latest
+    volumes:
+      - ./src/opensearch/pipelines.yaml:/usr/share/data-prepper/pipelines/pipelines.yaml
+      - ./src/opensearch/data-prepper-config.yaml:/usr/share/data-prepper/config/data-prepper-config.yaml
+    ports:
+      - "21892"
+    logging: *logging
+
   # *****
   # Tests
   # *****
diff --git a/src/grafana/provisioning/datasources/opensearch.yaml b/src/grafana/provisioning/datasources/opensearch.yaml
new file mode 100644
index 0000000000..e4ffcf58a7
--- /dev/null
+++ b/src/grafana/provisioning/datasources/opensearch.yaml
@@ -0,0 +1,20 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: 1
+
+datasources:
+  - name: OpenSearch
+    type: grafana-opensearch-datasource
+    url: http://opensearch:9200/
+    access: proxy
+    editable: true
+    isDefault: false
+    jsonData:
+      database: 'otel'
+      interval: Daily
+      timeField: 'time'
+      version: '2.10.0'
+      flavor: 'opensearch'
+      logMessageField: body
+      logLevelField: severitynumber
diff --git a/src/opensearch/data-prepper-config.yaml b/src/opensearch/data-prepper-config.yaml
new file mode 100644
index 0000000000..2af22a1b45
--- /dev/null
+++ b/src/opensearch/data-prepper-config.yaml
@@ -0,0 +1,4 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
+ssl: false
diff --git a/src/opensearch/pipelines.yaml b/src/opensearch/pipelines.yaml
new file mode 100644
index 0000000000..71acd227cb
--- /dev/null
+++ b/src/opensearch/pipelines.yaml
@@ -0,0 +1,14 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
+demo-pipeline:
+  source:
+    otel_logs_source:
+      ssl: false
+  sink:
+    - opensearch:
+        hosts: ["http://opensearch:9200"]
+        insecure: true
+        username: admin
+        password: admin
+        index: otel
diff --git a/src/otelcollector/otelcol-observability.yml b/src/otelcollector/otelcol-observability.yml
index fb4b14646c..52b7098f50 100644
--- a/src/otelcollector/otelcol-observability.yml
+++ b/src/otelcollector/otelcol-observability.yml
@@ -7,6 +7,10 @@ exporters:
     endpoint: "jaeger:4317"
     tls:
       insecure: true
+  otlp/logs:
+    endpoint: "dataprepper:21892"
+    tls:
+      insecure: true
   otlphttp/prometheus:
     endpoint: "http://prometheus:9090/api/v1/otlp"
     tls:
@@ -18,3 +22,5 @@ service:
       exporters: [otlp, logging, spanmetrics]
     metrics:
       exporters: [otlphttp/prometheus, logging]
+    logs:
+      exporters: [otlp/logs, logging]