[opentelemetry] Add OTLP intake E2E system tests

.github/workflows/ci.yml

@@ -613,6 +613,40 @@ jobs:
     - name: Print fancy log report
       if: ${{ always() }}
       run: python utils/scripts/markdown_logs.py >> $GITHUB_STEP_SUMMARY
+
+  test-the-tests-open-telemetry:
+    runs-on: ubuntu-latest
+    if: github.event.action != 'opened' && !contains(github.event.pull_request.labels.*.name, 'run-default-scenario')
+    needs:
+    - lint_and_test
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Setup python 3.9
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.9'
+    - name: Pull mitmproxy image
+      run: docker pull mitmproxy/mitmproxy
+    - name: Build
+      id: build
+      run: SYSTEM_TEST_BUILD_ATTEMPTS=3 ./build.sh java_otel
+    - name: Run OTEL_TRACING_E2E scenario
+      if: steps.build.outcome == 'success'
+      run: ./run.sh OTEL_TRACING_E2E
+      env:
+        DD_API_KEY: ${{ secrets.DD_API_KEY }}
+        DD_APPLICATION_KEY: ${{ secrets.DD_APPLICATION_KEY }}
+    - name: Compress logs
+      if: steps.build.outcome == 'success'
+      run: tar -czvf artifact.tar.gz $(ls | grep logs)
+    - name: Upload artifact
+      if: steps.build.outcome == 'success'
+      uses: actions/upload-artifact@v3
+      with:
+        name: logs_java-otel_spring-boot-native_prod
+        path: artifact.tar.gz
+
   post_test-the-tests:
     runs-on: ubuntu-latest
     needs:

.vscode/settings.json

@@ -4,5 +4,6 @@
     "python.testing.unittestEnabled": false,
     "python.linting.enabled": true,
     "python.linting.pylintEnabled": true,
-    "python.formatting.provider": "black"
+    "python.formatting.provider": "black",
+    "java.compile.nullAnalysis.mode": "disabled"
 }

requirements.txt

@@ -25,4 +25,6 @@ rfc3339-validator==0.1.4
 matplotlib
 
 docker==6.0.0
+
+opentelemetry-proto==1.17.0
 paramiko==3.1.0

tests/otel_tracing_e2e/_validator.py

@@ -0,0 +1,77 @@
+# Util functions to validate JSON trace data from OTel system tests
+
+import json
+
+
+def validate_trace(traces: list, use_128_bits_trace_id: bool):
+    server_span = None
+    message_span = None
+    for trace in traces:
+        spans = trace["spans"]
+        assert len(spans) == 1
+        for item in spans.items():
+            span = item[1]
+            validate_common_tags(span, use_128_bits_trace_id)
+            if span["type"] == "web":
+                server_span = span
+            elif span["type"] == "custom":
+                message_span = span
+            else:
+                raise Exception("Unexpected span ", span)
+    validate_server_span(server_span)
+    validate_message_span(message_span)
+    validate_span_link(server_span, message_span)
+
+
+def validate_common_tags(span: dict, use_128_bits_trace_id: bool):
+    expected_tags = {
+        "parent_id": "0",
+        "env": "system-tests",
+        "service": "otel-system-tests-spring-boot",
+        "ingestion_reason": "otel",
+    }
+    expected_meta = {
+        "env": "system-tests",
+        "deployment.environment": "system-tests",
+        "_dd.ingestion_reason": "otel",
+        "otel.status_code": "Unset",
+        "otel.user_agent": "OTel-OTLP-Exporter-Java/1.23.1",
+        "otel.library.name": "com.datadoghq.springbootnative",
+    }
+    assert expected_tags.items() <= span.items()
+    assert expected_meta.items() <= span["meta"].items()
+    validate_trace_id(span, use_128_bits_trace_id)
+
+
+def validate_trace_id(span: dict, use_128_bits_trace_id: bool):
+    dd_trace_id = int(span["trace_id"], base=10)
+    otel_trace_id = int(span["meta"]["otel.trace_id"], base=16)
+    if use_128_bits_trace_id:
+        assert dd_trace_id == otel_trace_id
+    else:
+        trace_id_bytes = otel_trace_id.to_bytes(16, "big")
+        assert dd_trace_id == int.from_bytes(trace_id_bytes[8:], "big")
+
+
+def validate_server_span(span: dict):
+    expected_tags = {"name": "WebController.home", "resource": "GET /"}
+    expected_meta = {"http.route": "/", "http.method": "GET"}
+    assert expected_tags.items() <= span.items()
+    assert expected_meta.items() <= span["meta"].items()
+
+
+def validate_message_span(span: dict):
+    expected_tags = {"name": "WebController.home.publish", "resource": "publish"}
+    expected_meta = {"messaging.operation": "publish", "messaging.system": "rabbitmq"}
+    assert expected_tags.items() <= span.items()
+    assert expected_meta.items() <= span["meta"].items()
+
+
+def validate_span_link(server_span: dict, message_span: dict):
+    span_links = json.loads(server_span["meta"]["_dd.span_links"])
+    assert len(span_links) == 1
+    span_link = span_links[0]
+    assert span_link["trace_id"] == message_span["meta"]["otel.trace_id"]
+    span_id_hex = f'{int(message_span["span_id"]):x}'  # span_id is an int in span but a hex in span_links
+    assert span_link["span_id"] == span_id_hex
+    assert span_link["attributes"] == {"messaging.operation": "publish"}

tests/otel_tracing_e2e/test_e2e.py

@@ -0,0 +1,25 @@
+from _validator import validate_trace
+from utils import context, weblog, interfaces, scenarios, irrelevant
+
+
+@scenarios.otel_tracing_e2e
+@irrelevant(context.library != "open_telemetry")
+class Test_OTel_E2E:
+    def setup_main(self):
+        self.use_128_bits_trace_id = False
+        self.r = weblog.get(path="/")
+
+    def test_main(self):
+        otel_trace_ids = set(interfaces.open_telemetry.get_otel_trace_id(request=self.r))
+        assert len(otel_trace_ids) == 2
+        dd_trace_ids = [self._get_dd_trace_id(otel_trace_id) for otel_trace_id in otel_trace_ids]
+        traces = [
+            interfaces.backend.assert_otlp_trace_exist(request=self.r, dd_trace_id=dd_trace_id)
+            for dd_trace_id in dd_trace_ids
+        ]
+        validate_trace(traces, self.use_128_bits_trace_id)
+
+    def _get_dd_trace_id(self, otel_trace_id=bytes) -> int:
+        if self.use_128_bits_trace_id:
+            return int.from_bytes(otel_trace_id, "big")
+        return int.from_bytes(otel_trace_id[8:], "big")

utils/_context/_scenarios.py

@@ -415,6 +415,100 @@ def get_junit_properties(self):
         return result
 
 
+class OpenTelemetryScenario(_DockerScenario):
+    """ Scenario for testing opentelemetry"""
+
+    def __init__(self, name, weblog_env) -> None:
+        self._required_containers = []
+        super().__init__(name, use_proxy=True)
+        if not self.is_current_scenario:
+            return
+
+        self.weblog_container = WeblogContainer(self.host_log_folder, environment=weblog_env)
+        self._required_containers.append(self.weblog_container)
+
+    def _create_interface_folders(self):
+        for interface in ("open_telemetry", "backend"):
+            self.create_log_subfolder(f"interfaces/{interface}")
+
+    def _start_interface_watchdog(self):
+        from utils import interfaces
+
+        class Event(FileSystemEventHandler):
+            def __init__(self, interface) -> None:
+                super().__init__()
+                self.interface = interface
+
+            def on_modified(self, event):
+                if event.is_directory:
+                    return
+
+                self.interface.ingest_file(event.src_path)
+
+        observer = Observer()
+        observer.schedule(
+            Event(interfaces.open_telemetry), path=f"{self.host_log_folder}/interfaces/open_telemetry", recursive=True
+        )
+
+        observer.start()
+
+    def _get_warmups(self):
+        warmups = super()._get_warmups()
+
+        warmups.insert(0, self._create_interface_folders)
+        warmups.insert(1, self._start_interface_watchdog)
+        warmups.append(self._wait_for_app_readiness)
+
+        return warmups
+
+    def _wait_for_app_readiness(self):
+        from utils import interfaces  # import here to avoid circular import
+
+        if self.use_proxy:
+            logger.debug("Wait for app readiness")
+
+            if not interfaces.open_telemetry.ready.wait(40):
+                raise Exception("Open telemetry interface not ready")
+            logger.debug("Open telemetry ready")
+
+    def post_setup(self, session):
+        from utils import interfaces
+
+        if self.use_proxy:
+            self._wait_interface(interfaces.open_telemetry, session, 5)
+
+            self.collect_logs()
+
+            self._wait_interface(interfaces.library_stdout, session, 0)
+            self._wait_interface(interfaces.library_dotnet_managed, session, 0)
+        else:
+            self.collect_logs()
+
+    @staticmethod
+    def _wait_interface(interface, session, timeout):
+        terminal = session.config.pluginmanager.get_plugin("terminalreporter")
+        terminal.write_sep("-", f"Wait for {interface} ({timeout}s)")
+        terminal.flush()
+
+        interface.wait(timeout)
+
+    @property
+    def library(self):
+        return LibraryVersion("open_telemetry", "0.0.0")
+
+    @property
+    def agent(self):
+        return LibraryVersion("agent", "0.0.0")
+
+    @property
+    def agent_version(self):
+        return self.agent.version
+
+    @property
+    def weblog_variant(self):
+        return self.weblog_container.weblog_variant
+
+
 class CgroupScenario(EndToEndScenario):
 
     # cgroup test
@@ -669,6 +763,11 @@ class scenarios:
         backend_interface_timeout=5,
     )
 
+    otel_tracing_e2e = OpenTelemetryScenario(
+        "OTEL_TRACING_E2E",
+        weblog_env={"DD_API_KEY": os.environ.get("DD_API_KEY"), "DD_SITE": os.environ.get("DD_SITE"),},
+    )
+
     library_conf_custom_headers_short = EndToEndScenario(
         "LIBRARY_CONF_CUSTOM_HEADERS_SHORT", additional_trace_header_tags=("header-tag1", "header-tag2")
     )

utils/build/build.sh

@@ -27,6 +27,7 @@ readonly DEFAULT_python=flask-poc
 readonly DEFAULT_ruby=rails70
 readonly DEFAULT_golang=net-http
 readonly DEFAULT_java=spring-boot
+readonly DEFAULT_java_otel=spring-boot-native
 readonly DEFAULT_php=apache-mod-8.0
 readonly DEFAULT_dotnet=poc
 readonly DEFAULT_cpp=nginx
@@ -256,7 +257,7 @@ COMMAND=build
 
 while [[ "$#" -gt 0 ]]; do
     case $1 in
-        cpp|dotnet|golang|java|nodejs|php|python|ruby) TEST_LIBRARY="$1";;
+        cpp|dotnet|golang|java|java_otel|nodejs|php|python|ruby) TEST_LIBRARY="$1";;
         -l|--library) TEST_LIBRARY="$2"; shift ;;
         -i|--images) BUILD_IMAGES="$2"; shift ;;
         -w|--weblog-variant) WEBLOG_VARIANT="$2"; shift ;;

utils/build/docker/java_otel/spring-boot-native.Dockerfile

@@ -0,0 +1,29 @@
+FROM openjdk:17-buster
+
+# Install required bsdtar
+RUN apt-get update && \
+	apt-get install -y libarchive-tools
+
+
+# Install maven
+RUN curl https://archive.apache.org/dist/maven/maven-3/3.8.6/binaries/apache-maven-3.8.6-bin.tar.gz --output /opt/maven.tar.gz && \
+	tar xzvf /opt/maven.tar.gz --directory /opt && \
+	rm /opt/maven.tar.gz
+
+WORKDIR /app
+
+# Copy application sources and cache dependencies
+COPY ./utils/build/docker/java_otel/spring-boot-native/pom.xml .
+COPY ./utils/build/docker/java_otel/spring-boot-native/src ./src
+
+# Compile application
+RUN /opt/apache-maven-3.8.6/bin/mvn clean package
+
+# Set up required args
+RUN echo "1.23.1" > SYSTEM_TESTS_LIBRARY_VERSION
+RUN echo "1.0.0" > SYSTEM_TESTS_LIBDDWAF_VERSION
+RUN echo "1.0.0" > SYSTEM_TESTS_APPSEC_EVENT_RULES_VERSION
+
+RUN echo "#!/bin/bash\njava -jar target/myproject-3.0.0-SNAPSHOT.jar --server.port=7777" > app.sh
+RUN chmod +x app.sh
+CMD [ "./app.sh" ]