[opentelemetry] Add OTLP intake E2E system tests

requirements.txt

@@ -19,3 +19,5 @@ rfc3339-validator==0.1.4
 matplotlib
 
 docker==6.0.0
+
+opentelemetry-proto==1.17.0

tests/otel_tracing_e2e/_validator.py

@@ -0,0 +1,77 @@
+# Util functions to validate JSON trace data from OTel system tests
+
+import json
+
+
+def validate_trace(traces: list, use_128_bits_trace_id: bool):
+    server_span = None
+    message_span = None
+    for trace in traces:
+        spans = trace["spans"]
+        assert len(spans) == 1
+        for item in spans.items():
+            span = item[1]
+            validate_common_tags(span, use_128_bits_trace_id)
+            if span["type"] == "web":
+                server_span = span
+            elif span["type"] == "custom":
+                message_span = span
+            else:
+                raise Exception("Unexpected span ", span)
+    validate_server_span(server_span)
+    validate_message_span(message_span)
+    validate_span_link(server_span, message_span)
+
+
+def validate_common_tags(span: dict, use_128_bits_trace_id: bool):
+    expected_tags = {
+        "parent_id": "0",
+        "env": "system-tests",
+        "service": "otel-system-tests-spring-boot",
+        "ingestion_reason": "otel",
+    }
+    expected_meta = {
+        "env": "system-tests",
+        "deployment.environment": "system-tests",
+        "_dd.ingestion_reason": "otel",
+        "otel.status_code": "Unset",
+        "otel.user_agent": "OTel-OTLP-Exporter-Java/1.23.1",
+        "otel.library.name": "com.datadoghq.springbootnative",
+    }
+    assert expected_tags.items() <= span.items()
+    assert expected_meta.items() <= span["meta"].items()
+    validate_trace_id(span, use_128_bits_trace_id)
+
+
+def validate_trace_id(span: dict, use_128_bits_trace_id: bool):
+    dd_trace_id = int(span["trace_id"], base=10)
+    otel_trace_id = int(span["meta"]["otel.trace_id"], base=16)
+    if use_128_bits_trace_id:
+        assert dd_trace_id == otel_trace_id
+    else:
+        trace_id_bytes = otel_trace_id.to_bytes(16, "big")
+        assert dd_trace_id == int.from_bytes(trace_id_bytes[8:], "big")
+
+
+def validate_server_span(span: dict):
+    expected_tags = {"name": "WebController.home", "resource": "GET /"}
+    expected_meta = {"http.route": "/", "http.method": "GET"}
+    assert expected_tags.items() <= span.items()
+    assert expected_meta.items() <= span["meta"].items()
+
+
+def validate_message_span(span: dict):
+    expected_tags = {"name": "WebController.home.publish", "resource": "publish"}
+    expected_meta = {"messaging.operation": "publish", "messaging.system": "rabbitmq"}
+    assert expected_tags.items() <= span.items()
+    assert expected_meta.items() <= span["meta"].items()
+
+
+def validate_span_link(server_span: dict, message_span: dict):
+    span_links = json.loads(server_span["meta"]["_dd.span_links"])
+    assert len(span_links) == 1
+    span_link = span_links[0]
+    assert span_link["trace_id"] == message_span["meta"]["otel.trace_id"]
+    span_id_hex = f'{int(message_span["span_id"]):x}'  # span_id is an int in span but a hex in span_links
+    assert span_link["span_id"] == span_id_hex
+    assert span_link["attributes"] == {"messaging.operation": "publish"}

tests/otel_tracing_e2e/test_e2e.py

@@ -0,0 +1,27 @@
+from _validator import validate_trace
+from utils import context, weblog, interfaces, scenarios, missing_feature
+
+
+@scenarios.otel_tracing_e2e
+@missing_feature(
+    context.library != "java_otel", reason="OTel tests only support OTel instrumented applications at the moment.",
+)
+class Test_OTel_E2E:
+    def setup_main(self):
+        self.use_128_bits_trace_id = False
+        self.r = weblog.get(path="/")
+
+    def test_main(self):
+        otel_trace_ids = list(interfaces.library.get_otel_trace_id(request=self.r))
+        assert len(otel_trace_ids) == 2
+        dd_trace_ids = map(self._get_dd_trace_id, otel_trace_ids)
+        traces = map(
+            lambda dd_trace_id: interfaces.backend.assert_otlp_trace_exist(request=self.r, dd_trace_id=dd_trace_id),
+            dd_trace_ids,
+        )
+        validate_trace(traces, self.use_128_bits_trace_id)
+
+    def _get_dd_trace_id(self, otel_trace_id=bytes) -> int:
+        if self.use_128_bits_trace_id:
+            return int.from_bytes(otel_trace_id, "big")
+        return int.from_bytes(otel_trace_id[8:], "big")

utils/_context/_scenarios.py

@@ -512,6 +512,12 @@ class scenarios:
         backend_interface_timeout=5,
     )
 
+    # OpenTelemetry tracing end-to-end scenarios
+    otel_tracing_e2e = EndToEndScenario(
+        "OTEL_TRACING_E2E",
+        weblog_env={"DD_API_KEY": os.environ.get("DD_API_KEY"), "DD_SITE": os.environ.get("DD_SITE"),},
+    )
+
     library_conf_custom_headers_short = EndToEndScenario(
         "LIBRARY_CONF_CUSTOM_HEADERS_SHORT", additional_trace_header_tags=("header-tag1", "header-tag2")
     )

utils/build/build.sh

@@ -27,6 +27,7 @@ readonly DEFAULT_python=flask-poc
 readonly DEFAULT_ruby=rails70
 readonly DEFAULT_golang=net-http
 readonly DEFAULT_java=spring-boot
+readonly DEFAULT_java_otel=spring-boot-native
 readonly DEFAULT_php=apache-mod-8.0
 readonly DEFAULT_dotnet=poc
 readonly DEFAULT_cpp=nginx
@@ -252,7 +253,7 @@ COMMAND=build
 
 while [[ "$#" -gt 0 ]]; do
     case $1 in
-        cpp|dotnet|golang|java|nodejs|php|python|ruby) TEST_LIBRARY="$1";;
+        cpp|dotnet|golang|java|java_otel|nodejs|php|python|ruby) TEST_LIBRARY="$1";;
         -l|--library) TEST_LIBRARY="$2"; shift ;;
         -i|--images) BUILD_IMAGES="$2"; shift ;;
         -w|--weblog-variant) WEBLOG_VARIANT="$2"; shift ;;

utils/build/docker/java_otel/spring-boot-native.Dockerfile

@@ -0,0 +1,29 @@
+FROM openjdk:17-buster
+
+# Install required bsdtar
+RUN apt-get update && \
+	apt-get install -y libarchive-tools
+
+
+# Install maven
+RUN curl https://archive.apache.org/dist/maven/maven-3/3.8.6/binaries/apache-maven-3.8.6-bin.tar.gz --output /opt/maven.tar.gz && \
+	tar xzvf /opt/maven.tar.gz --directory /opt && \
+	rm /opt/maven.tar.gz
+
+WORKDIR /app
+
+# Copy application sources and cache dependencies
+COPY ./utils/build/docker/java_otel/spring-boot-native/pom.xml .
+COPY ./utils/build/docker/java_otel/spring-boot-native/src ./src
+
+# Compile application
+RUN /opt/apache-maven-3.8.6/bin/mvn clean package
+
+# Set up required args
+RUN echo "1.23.1" > SYSTEM_TESTS_LIBRARY_VERSION
+RUN echo "1.0.0" > SYSTEM_TESTS_LIBDDWAF_VERSION
+RUN echo "1.0.0" > SYSTEM_TESTS_APPSEC_EVENT_RULES_VERSION
+
+RUN echo "#!/bin/bash\njava -jar target/myproject-3.0.0-SNAPSHOT.jar --server.port=7777" > app.sh
+RUN chmod +x app.sh
+CMD [ "./app.sh" ]

utils/build/docker/java_otel/spring-boot-native/pom.xml

@@ -0,0 +1,94 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.springframework.boot</groupId>
+        <artifactId>spring-boot-starter-parent</artifactId>
+        <version>3.0.1</version>
+    </parent>
+
+    <groupId>com.example</groupId>
+    <artifactId>myproject</artifactId>
+    <version>3.0.0-SNAPSHOT</version>
+    <packaging>jar</packaging>
+
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>io.opentelemetry</groupId>
+                <artifactId>opentelemetry-bom</artifactId>
+                <version>1.23.1</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-tomcat</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>javax.servlet</groupId>
+            <artifactId>javax.servlet-api</artifactId>
+            <version>4.0.1</version>
+        </dependency>
+
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-api</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-sdk</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-exporter-otlp</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-exporter-logging-otlp</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-exporter-logging-otlp</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-extension-trace-propagators</artifactId>
+            <version>1.24.0</version>
+        </dependency>
+
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-semconv</artifactId>
+            <version>1.23.1-alpha</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

utils/build/docker/java_otel/spring-boot-native/src/main/java/com/datadoghq/springbootnative/App.java

@@ -0,0 +1,53 @@
+package com.datadoghq.springbootnative;
+
+import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.exporter.logging.otlp.OtlpJsonLoggingSpanExporter;
+import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporter;
+import io.opentelemetry.sdk.OpenTelemetrySdk;
+import io.opentelemetry.sdk.resources.Resource;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.SpanProcessor;
+import io.opentelemetry.sdk.trace.export.BatchSpanProcessor;
+import io.opentelemetry.sdk.trace.export.SpanExporter;
+import io.opentelemetry.sdk.trace.samplers.Sampler;
+import io.opentelemetry.semconv.resource.attributes.ResourceAttributes;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.annotation.ComponentScan;
+
+@SpringBootApplication
+@ComponentScan(basePackages = {"com.datadoghq.springbootnative"})
+public class App {
+    public static void main(String[] args) {
+        Resource resource = Resource.create(Attributes.of(
+                ResourceAttributes.SERVICE_NAME, "otel-system-tests-spring-boot",
+                ResourceAttributes.DEPLOYMENT_ENVIRONMENT, "system-tests"));
+
+        OtlpHttpSpanExporter intakeExporter = OtlpHttpSpanExporter.builder()
+                .setEndpoint("http://runner:8126/api/v0.2/traces")  // send to the proxy first
+                .addHeader("dd-protocol", "otlp")
+                .addHeader("dd-api-key", System.getenv("DD_API_KEY"))
+                .build();
+
+        SpanExporter loggingSpanExporter = OtlpJsonLoggingSpanExporter.create();
+
+        SpanExporter exporter = SpanExporter.composite(intakeExporter, loggingSpanExporter);
+
+        SpanProcessor processor = BatchSpanProcessor.builder(exporter)
+                .setMaxExportBatchSize(1)
+                .build();
+
+        SdkTracerProvider sdkTracerProvider = SdkTracerProvider.builder()
+                .addSpanProcessor(processor)
+                .setSampler(Sampler.alwaysOn())
+                .setResource(resource)
+                .build();
+
+        OpenTelemetry openTelemetry = OpenTelemetrySdk.builder()
+                .setTracerProvider(sdkTracerProvider)
+                .buildAndRegisterGlobal();
+
+        SpringApplication.run(App.class, args);
+    }
+}

utils/build/docker/java_otel/spring-boot-native/src/main/java/com/datadoghq/springbootnative/WebController.java

@@ -0,0 +1,54 @@
+package com.datadoghq.springbootnative;
+
+import io.opentelemetry.api.GlobalOpenTelemetry;
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.SpanContext;
+import io.opentelemetry.api.trace.SpanKind;
+import io.opentelemetry.api.trace.Tracer;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.context.Scope;
+import io.opentelemetry.semconv.trace.attributes.SemanticAttributes;
+import org.springframework.http.HttpHeaders;
+import org.springframework.web.bind.annotation.RequestHeader;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+public class WebController {
+  private final Tracer tracer = GlobalOpenTelemetry.getTracer("com.datadoghq.springbootnative");
+
+  @RequestMapping("/")
+  private String home(@RequestHeader HttpHeaders headers) throws InterruptedException {
+    try (Scope scope = Context.current().makeCurrent()) {
+      SpanContext spanContext = fakeAsyncWork(headers);
+      Span span = tracer.spanBuilder("WebController.home")
+              .setSpanKind(SpanKind.SERVER)
+              .addLink(spanContext, Attributes.of(AttributeKey.stringKey("messaging.operation"), "publish"))
+              .setAttribute(SemanticAttributes.HTTP_ROUTE, "/")
+              .setAttribute(SemanticAttributes.HTTP_METHOD, "GET")
+              .setAttribute("http.request.headers.user-agent", headers.get("User-Agent").get(0))
+              .startSpan();
+      try (Scope ignored = span.makeCurrent()) {
+        Thread.sleep(5);
+        return "Hello World!";
+      } finally {
+        span.end();
+      }
+    }
+  }
+
+  // Create a fake producer span and return its span context to test span links
+  private SpanContext fakeAsyncWork(HttpHeaders headers) throws InterruptedException {
+    Span fakeSpan = tracer.spanBuilder("WebController.home.publish")
+            .setSpanKind(SpanKind.PRODUCER)
+            .setAttribute("messaging.system", "rabbitmq")
+            .setAttribute("messaging.operation", "publish")
+            .setAttribute("http.request.headers.user-agent", headers.get("User-Agent").get(0))
+            .startSpan();
+    Thread.sleep(1);
+    fakeSpan.end();
+    return fakeSpan.getSpanContext();
+  }
+}

utils/interfaces/_backend.py

@@ -65,6 +65,19 @@ def assert_library_traces_exist(self, request, min_traces_len=1):
         ), f"We only found {len(traces)} traces in the library (tracers), but we expected {min_traces_len}!"
         return traces
 
+    def assert_otlp_trace_exist(self, request: requests.Request, dd_trace_id: str) -> dict:
+        """Attempts to fetch from the backend, ALL the traces that the OpenTelemetry SDKs sent to Datadog
+        during the execution of the given request.
+
+        The assosiation of the traces with a request is done through propagating the request ID (inside user agent)
+        on all the submitted traces. This is done automatically, unless you create root spans manually, which in
+        that case you need to manually propagate the user agent to the new spans.
+        """
+
+        rid = get_rid_from_request(request)
+        data = self._wait_for_trace(rid=rid, trace_id=dd_trace_id, retries=5, sleep_interval_multiplier=2.0)
+        return json.loads(data["response"]["content"])["trace"]
+
     def assert_single_spans_exist(self, request, min_spans_len=1, limit=100):
         """Attempts to fetch single span events using the given `query_filter` as part of the search query.
         The query should be what you would use in the `/apm/traces` page in the UI.