[opentelemetry] Add OTLP intake E2E system tests

requirements.txt

@@ -19,3 +19,9 @@ rfc3339-validator==0.1.4
 matplotlib
 
 docker==6.0.0
+
+opentelemetry-api==1.16.0
+opentelemetry-exporter-otlp-proto-http==1.16.0
+opentelemetry-sdk==1.16.0
+opentelemetry-propagator-b3==1.16.0
+opentelemetry-semantic-conventions==0.37b0

tests/otel_tracing_e2e/_validator.py

@@ -0,0 +1,93 @@
+import json
+
+# Util functions to validate JSON trace data from OTel system tests
+
+
+def validate_trace(trace_data, dd_trace_id, root_span_id, otel_trace_id):
+    assert root_span_id == int(trace_data["root_id"])
+    spans = trace_data["spans"]
+    assert len(spans) == 3
+    root_span = None
+    server_span = None
+    message_span = None
+    for item in spans.items():
+        span_id = item[0]
+        span = item[1]
+        validate_common_tags(span, dd_trace_id, otel_trace_id)
+        if int(span_id) == root_span_id:
+            root_span = span
+        elif span["type"] == "web":
+            server_span = span
+        elif span["type"] == "custom":
+            message_span = span
+        else:
+            raise Exception("Unexpected span ", span)
+    validate_root_span(root_span, server_span["span_id"], message_span["span_id"])
+    validate_server_span(server_span, root_span_id)
+    validate_message_span(message_span, root_span_id)
+    # TODO: validate span links once the format is fixed in intake
+    # validate_span_link(server_span, message_span, otel_trace_id)
+
+
+def validate_common_tags(span, dd_trace_id, otel_trace_id):
+    assert int(span["trace_id"]) == dd_trace_id
+    assert int(span["meta"]["otel.trace_id"], base=16) == otel_trace_id
+    assert span["env"] == "system-tests"
+    assert span["meta"]["env"] == "system-tests"
+    assert span["meta"]["deployment.environment"] == "system-tests"
+    assert span["meta"]["_dd.ingestion_reason"] == "otel"
+    assert span["ingestion_reason"] == "otel"
+    assert span["meta"]["otel.status_code"] == "STATUS_CODE_UNSET"
+
+
+def validate_root_span(span, server_span_id, message_span_id):
+    assert span["parent_id"] == "0"
+    assert span["type"] == "http"
+    assert span["service"] == "system-tests-runner"
+    assert span["name"] == "runner.get"
+    assert span["resource"] == "GET"
+    assert span["meta"]["otel.user_agent"] == "OTel-OTLP-Exporter-Python/1.16.0"
+    assert span["meta"]["otel.library.name"] == "system-tests-runner"
+    assert span["meta"]["telemetry.sdk.name"] == "opentelemetry"
+    assert span["meta"]["telemetry.sdk.language"] == "python"
+    assert span["meta"]["telemetry.sdk.version"] == "1.16.0"
+    assert span["meta"]["http.status_code"] == "200"
+    assert span["meta"]["http.host"] == "weblog"
+    assert span["meta"]["http.url"] == "http://weblog:7777"
+    assert span["meta"]["http.method"] == "GET"
+    assert len(span["children_ids"]) == 2
+    assert server_span_id in span["children_ids"]
+    assert message_span_id in span["children_ids"]
+
+
+def validate_server_span(span, root_span_id):
+    assert int(span["parent_id"]) == root_span_id
+    assert span["type"] == "web"
+    assert span["service"] == "otel-system-tests-spring-boot"
+    assert span["name"] == "WebController.home"
+    assert span["resource"] == "GET /"
+    assert span["meta"]["otel.user_agent"] == "OTel-OTLP-Exporter-Java/1.23.1"
+    assert span["meta"]["otel.library.name"] == "com.datadoghq.springbootnative"
+    assert span["meta"]["http.route"] == "/"
+    assert span["meta"]["http.method"] == "GET"
+
+
+def validate_message_span(span, root_span_id):
+    assert int(span["parent_id"]) == root_span_id
+    assert span["type"] == "custom"
+    assert span["service"] == "otel-system-tests-spring-boot"
+    assert span["name"] == "WebController.home.publish"
+    assert span["resource"] == "publish"
+    assert span["meta"]["otel.user_agent"] == "OTel-OTLP-Exporter-Java/1.23.1"
+    assert span["meta"]["otel.library.name"] == "com.datadoghq.springbootnative"
+    assert span["meta"]["messaging.operation"] == "publish"
+    assert span["meta"]["messaging.system"] == "rabbitmq"
+
+
+def validate_span_link(server_span, message_span, otel_trace_id):
+    span_links = json.load(server_span["meta"]["_dd.span_links"])
+    assert len(span_links) == 1
+    span_link = span_links[0]
+    assert int(span_link["trace_id"], base=16) == otel_trace_id
+    assert span_link["span_id"] == message_span["span_id"]
+    assert span_link["attributes"] == {"messaging.operation": "publish"}

tests/otel_tracing_e2e/test_e2e.py

@@ -0,0 +1,83 @@
+import json
+import os
+
+from opentelemetry import trace, propagate
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk.resources import DEPLOYMENT_ENVIRONMENT, Resource, SERVICE_NAME
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.semconv.trace import SpanAttributes
+from opentelemetry.propagators.b3 import B3MultiFormat, B3SingleFormat
+from _validator import validate_trace
+from utils import context, weblog, interfaces, scenarios, missing_feature
+
+
+class E2ETestBase:
+    def setup_main(self):
+        self.setup_opentelemetry()
+        self.use_128_bits_trace_id = False
+        with self.tracer.start_as_current_span(name="runner.get", kind=trace.SpanKind.CLIENT) as span:
+            span.set_attribute(SpanAttributes.HTTP_METHOD, "GET")
+            span.set_attribute(SpanAttributes.HTTP_URL, "http://weblog:7777")
+            headers = {}
+            propagate.get_global_textmap().inject(headers)
+            self.r = weblog.get(path="/", headers=headers)
+            span.set_attribute(SpanAttributes.HTTP_STATUS_CODE, self.r.status_code)
+            self.trace_id = span.get_span_context().trace_id
+            self.root_span_id = span.get_span_context().span_id
+
+    def setup_opentelemetry(self):
+        resource = Resource.create(
+            attributes={SERVICE_NAME: "system-tests-runner", DEPLOYMENT_ENVIRONMENT: "system-tests"}
+        )
+        dd_site = os.environ.get("DD_SITE")
+        if dd_site is None or dd_site == "":
+            dd_site = "datad0g.com"
+        exporter = OTLPSpanExporter(
+            endpoint=f"https://trace.agent.{dd_site}/api/v0.2/traces",
+            headers={"dd-protocol": "otlp", "dd-api-key": os.environ.get("DD_API_KEY"),},
+        )
+        processor = BatchSpanProcessor(span_exporter=exporter, max_export_batch_size=1)
+        trace.set_tracer_provider(TracerProvider(resource=resource, active_span_processor=processor))
+        self.tracer = trace.get_tracer("system-tests-runner")
+
+    def test_main(self):
+        response = interfaces.backend._wait_for_trace(
+            rid=self.r, trace_id=self._get_dd_trace_id(), retries=5, sleep_interval_multiplier=2.0
+        )
+        trace_data = json.loads(response["response"]["content"])["trace"]
+        validate_trace(trace_data, self._get_dd_trace_id(), self.root_span_id, self.trace_id)
+
+    def _get_dd_trace_id(self):
+        if self.use_128_bits_trace_id:
+            return self.trace_id
+        trace_id_bytes = self.trace_id.to_bytes(16, "big")
+        return int.from_bytes(trace_id_bytes[8:], "big")
+
+
+@scenarios.otel_tracing_e2e_w3c
+@missing_feature(
+    context.library != "java_otel", reason="OTel tests only support OTel instrumented applications at the moment.",
+)
+class Test_E2E_W3C(E2ETestBase):
+    pass  # Use the default propagator TraceContextTextMapPropagator (W3C propagator)
+
+
+@scenarios.otel_tracing_e2e_b3
+@missing_feature(
+    context.library != "java_otel", reason="OTel tests only support OTel instrumented applications at the moment.",
+)
+class Test_E2E_B3(E2ETestBase):
+    def setup_main(self):
+        propagate.set_global_textmap(B3SingleFormat())
+        E2ETestBase.setup_main(self)
+
+
+@scenarios.otel_tracing_e2e_b3_multi
+@missing_feature(
+    context.library != "java_otel", reason="OTel tests only support OTel instrumented applications at the moment.",
+)
+class Test_E2E_B3_Multi(E2ETestBase):
+    def setup_main(self):
+        propagate.set_global_textmap(B3MultiFormat())
+        E2ETestBase.setup_main(self)

utils/_context/_scenarios.py

@@ -197,11 +197,10 @@ def _get_warmups(self):
         for container in self._required_containers:
             warmups.append(container.start)
 
-        warmups += [
-            self.agent_container.start,
-            self.weblog_container.start,
-            EndToEndScenario._wait_for_app_readiness,
-        ]
+        warmups += [self.agent_container.start, self.weblog_container.start]
+        # TODO: Support Agent ingestion path in OTel tests
+        if self.weblog_container.library != "java_otel":
+            warmups += [EndToEndScenario._wait_for_app_readiness]
 
         return warmups
 
@@ -465,6 +464,28 @@ class scenarios:
         backend_interface_timeout=5,
     )
 
+    # OpenTelemetry tracing end-to-end scenarios
+    otel_tracing_e2e_w3c = EndToEndScenario(
+        "OTEL_TRACING_E2E_W3C",
+        weblog_env={"DD_API_KEY": os.environ.get("DD_API_KEY"), "DD_SITE": os.environ.get("DD_SITE"),},
+    )
+    otel_tracing_e2e_b3 = EndToEndScenario(
+        "OTEL_TRACING_E2E_B3",
+        weblog_env={
+            "DD_API_KEY": os.environ.get("DD_API_KEY"),
+            "DD_SITE": os.environ.get("DD_SITE"),
+            "OTEL_PROPAGATORS": "b3",
+        },
+    )
+    otel_tracing_e2e_b3_multi = EndToEndScenario(
+        "OTEL_TRACING_E2E_B3_MULTI",
+        weblog_env={
+            "DD_API_KEY": os.environ.get("DD_API_KEY"),
+            "DD_SITE": os.environ.get("DD_SITE"),
+            "OTEL_PROPAGATORS": "b3multi",
+        },
+    )
+
     library_conf_custom_headers_short = EndToEndScenario(
         "LIBRARY_CONF_CUSTOM_HEADERS_SHORT", additional_trace_header_tags=("header-tag1", "header-tag2")
     )

utils/build/build.sh

@@ -27,6 +27,7 @@ readonly DEFAULT_python=flask-poc
 readonly DEFAULT_ruby=rails70
 readonly DEFAULT_golang=net-http
 readonly DEFAULT_java=spring-boot
+readonly DEFAULT_java_otel=spring-boot-native
 readonly DEFAULT_php=apache-mod-8.0
 readonly DEFAULT_dotnet=poc
 readonly DEFAULT_cpp=nginx
@@ -213,7 +214,7 @@ COMMAND=build
 
 while [[ "$#" -gt 0 ]]; do
     case $1 in
-        cpp|dotnet|golang|java|nodejs|php|python|ruby) TEST_LIBRARY="$1";;
+        cpp|dotnet|golang|java|java_otel|nodejs|php|python|ruby) TEST_LIBRARY="$1";;
         -l|--library) TEST_LIBRARY="$2"; shift ;;
         -i|--images) BUILD_IMAGES="$2"; shift ;;
         -w|--weblog-variant) WEBLOG_VARIANT="$2"; shift ;;

utils/build/docker/java_otel/spring-boot-native.Dockerfile

@@ -0,0 +1,29 @@
+FROM openjdk:17-buster
+
+# Install required bsdtar
+RUN apt-get update && \
+	apt-get install -y libarchive-tools
+
+
+# Install maven
+RUN curl https://archive.apache.org/dist/maven/maven-3/3.8.6/binaries/apache-maven-3.8.6-bin.tar.gz --output /opt/maven.tar.gz && \
+	tar xzvf /opt/maven.tar.gz --directory /opt && \
+	rm /opt/maven.tar.gz
+
+WORKDIR /app
+
+# Copy application sources and cache dependencies
+COPY ./utils/build/docker/java_otel/spring-boot-native/pom.xml .
+COPY ./utils/build/docker/java_otel/spring-boot-native/src ./src
+
+# Compile application
+RUN /opt/apache-maven-3.8.6/bin/mvn clean package
+
+# Set up required args
+RUN echo "1.23.1" > SYSTEM_TESTS_LIBRARY_VERSION
+RUN echo "1.0.0" > SYSTEM_TESTS_LIBDDWAF_VERSION
+RUN echo "1.0.0" > SYSTEM_TESTS_APPSEC_EVENT_RULES_VERSION
+
+RUN echo "#!/bin/bash\njava -jar target/myproject-3.0.0-SNAPSHOT.jar --server.port=7777" > app.sh
+RUN chmod +x app.sh
+CMD [ "./app.sh" ]

utils/build/docker/java_otel/spring-boot-native/pom.xml

@@ -0,0 +1,94 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.springframework.boot</groupId>
+        <artifactId>spring-boot-starter-parent</artifactId>
+        <version>3.0.1</version>
+    </parent>
+
+    <groupId>com.example</groupId>
+    <artifactId>myproject</artifactId>
+    <version>3.0.0-SNAPSHOT</version>
+    <packaging>jar</packaging>
+
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>io.opentelemetry</groupId>
+                <artifactId>opentelemetry-bom</artifactId>
+                <version>1.23.1</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-tomcat</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>javax.servlet</groupId>
+            <artifactId>javax.servlet-api</artifactId>
+            <version>4.0.1</version>
+        </dependency>
+
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-api</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-sdk</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-exporter-otlp</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-exporter-logging-otlp</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-exporter-logging-otlp</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-extension-trace-propagators</artifactId>
+            <version>1.24.0</version>
+        </dependency>
+
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-semconv</artifactId>
+            <version>1.23.1-alpha</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>