From 765a887ddc7703dea0c2f22bd6b9eb2f4e814a04 Mon Sep 17 00:00:00 2001 From: Yang Song Date: Mon, 27 Mar 2023 16:14:34 -0400 Subject: [PATCH 1/9] [opentelemetry] Extend OTLP E2E system tests to DD Agent and OTel Collector ingestion paths --- .github/workflows/ci.yml | 6 +- requirements.txt | 1 + tests/otel_tracing_e2e/_validator.py | 87 +++++++++++++++++-- tests/otel_tracing_e2e/test_e2e.py | 39 ++++++++- utils/_context/_scenarios.py | 18 ++-- utils/_context/containers.py | 30 ++++++- utils/build/docker/agent.Dockerfile | 10 +++ .../com/datadoghq/springbootnative/App.java | 14 +-- utils/build/docker/otelcol-config.yaml | 30 +++++++ utils/interfaces/_backend.py | 32 ++++--- utils/interfaces/_open_telemetry.py | 6 +- utils/proxy/core.py | 17 +++- 12 files changed, 247 insertions(+), 43 deletions(-) create mode 100644 utils/build/docker/otelcol-config.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8d5c89b7b..e1d960c7a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -640,7 +640,11 @@ jobs: run: ./run.sh OTEL_TRACING_E2E env: DD_API_KEY: ${{ secrets.DD_API_KEY }} - DD_APPLICATION_KEY: ${{ secrets.DD_APPLICATION_KEY }} + DD_APP_KEY: ${{ secrets.DD_APPLICATION_KEY }} + DD_API_KEY_2: ${{ secrets.DD_API_KEY_2 }} + DD_APP_KEY_2: ${{ secrets.DD_APPLICATION_KEY_2 }} + DD_API_KEY_3: ${{ secrets.DD_API_KEY_3 }} + DD_APP_KEY_3: ${{ secrets.DD_APPLICATION_KEY_3 }} - name: Compress logs if: steps.build.outcome == 'success' run: tar -czvf artifact.tar.gz $(ls | grep logs) diff --git a/requirements.txt b/requirements.txt index 11c01df34e..032a3b2aa4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,3 +28,4 @@ docker==6.0.0 opentelemetry-proto==1.17.0 paramiko==3.1.0 +dictdiffer==0.9.0 diff --git a/tests/otel_tracing_e2e/_validator.py b/tests/otel_tracing_e2e/_validator.py index fb560a71cd..365134122e 100644 --- a/tests/otel_tracing_e2e/_validator.py +++ b/tests/otel_tracing_e2e/_validator.py @@ -1,9 +1,20 @@ # Util functions to validate JSON trace data from OTel system tests import json +import dictdiffer +# Validates traces from Agent, Collector and Backend intake OTLP ingestion paths are consistent +def validate_all_traces( + traces_agent: list[dict], traces_intake: list[dict], traces_collector: list[dict], use_128_bits_trace_id: bool +): + spans_agent = validate_trace(traces_agent, use_128_bits_trace_id) + spans_intake = validate_trace(traces_intake, use_128_bits_trace_id) + spans_collector = validate_trace(traces_collector, use_128_bits_trace_id) + validate_spans_from_all_paths(spans_agent, spans_intake, spans_collector) -def validate_trace(traces: list, use_128_bits_trace_id: bool): + +# Validates fields that we know the values upfront for one single trace from an OTLP ingestion path +def validate_trace(traces: list[dict], use_128_bits_trace_id: bool) -> tuple: server_span = None message_span = None for trace in traces: @@ -21,6 +32,7 @@ def validate_trace(traces: list, use_128_bits_trace_id: bool): validate_server_span(server_span) validate_message_span(message_span) validate_span_link(server_span, message_span) + return (server_span, message_span) def validate_common_tags(span: dict, use_128_bits_trace_id: bool): @@ -35,7 +47,6 @@ def validate_common_tags(span: dict, use_128_bits_trace_id: bool): "deployment.environment": "system-tests", "_dd.ingestion_reason": "otel", "otel.status_code": "Unset", - "otel.user_agent": "OTel-OTLP-Exporter-Java/1.23.1", "otel.library.name": "com.datadoghq.springbootnative", } assert expected_tags.items() <= span.items() @@ -68,10 +79,68 @@ def validate_message_span(span: dict): def validate_span_link(server_span: dict, message_span: dict): - span_links = json.loads(server_span["meta"]["_dd.span_links"]) - assert len(span_links) == 1 - span_link = span_links[0] - assert span_link["trace_id"] == message_span["meta"]["otel.trace_id"] - span_id_hex = f'{int(message_span["span_id"]):x}' # span_id is an int in span but a hex in span_links - assert span_link["span_id"] == span_id_hex - assert span_link["attributes"] == {"messaging.operation": "publish"} + # TODO: enable check on span links once newer version of Agent is used in system tests + if "_dd.span_links" not in server_span["meta"]: + return + + actual = json.loads(server_span["meta"]["_dd.span_links"]) + expected = [ + { + "trace_id": message_span["meta"]["otel.trace_id"], + "span_id": f'{int(message_span["span_id"]):x}', + "attributes": {"messaging.operation": "publish"}, + } + ] + assert actual == expected + + +# Validates fields that we don't know the values upfront for all 3 ingestion paths +def validate_spans_from_all_paths(spans_agent: tuple, spans_intake: tuple, spans_collector: tuple): + validate_span_fields(spans_agent[0], spans_intake[0], "Agent server span", "Intake server span") + validate_span_fields(spans_agent[0], spans_collector[0], "Agent server span", "Collector server span") + validate_span_fields(spans_agent[1], spans_intake[1], "Agent message span", "Intake message span") + validate_span_fields(spans_agent[1], spans_collector[1], "Agent message span", "Intake message span") + + +def validate_span_fields(span1: dict, span2: dict, name1: str, name2: str): + assert span1["start"] == span2["start"] + assert span1["end"] == span2["end"] + assert span1["duration"] == span2["duration"] + assert span1["resource_hash"] == span2["resource_hash"] + validate_span_metas_metrics(span1["meta"], span2["meta"], span1["metrics"], span2["metrics"], name1, name2) + + +KNOWN_UNMATCHED_METAS = [ + "otel.user_agent", + "span.kind", + "_dd.agent_version", + "_dd.span_links", # TODO: remove once Agent supports span links + "_dd.agent_rare_sampler.enabled", + "_dd.hostname", + "_dd.compute_stats", + "_dd.tracer_version", + "_dd.p.dm", + "_dd.agent_hostname", +] +KNOWN_UNMATCHED_METRICS = [ + "_dd.agent_errors_sampler.target_tps", + "_dd.agent_priority_sampler.target_tps", + "_sampling_priority_rate_v1", + "_dd.otlp_sr", +] + + +def validate_span_metas_metrics(meta1: dict, meta2: dict, metrics1: dict, metrics2: dict, name1: str, name2: str): + # Exclude fields that are expected to have different values for different ingestion paths + for known_unmatched_meta in KNOWN_UNMATCHED_METAS: + meta1.pop(known_unmatched_meta, None) + meta2.pop(known_unmatched_meta, None) + for known_unmatched_metric in KNOWN_UNMATCHED_METRICS: + metrics1.pop(known_unmatched_metric, None) + metrics2.pop(known_unmatched_metric, None) + + # Other fields should match + assert meta1 == meta2, f"Diff in metas between {name1} and {name2}: {list(dictdiffer.diff(meta1, meta2))}" + assert ( + metrics1 == metrics2 + ), f"Diff in metrics between {name1} and {name2}: {list(dictdiffer.diff(metrics1, metrics2))}" diff --git a/tests/otel_tracing_e2e/test_e2e.py b/tests/otel_tracing_e2e/test_e2e.py index 94cf490d16..0581a73a00 100644 --- a/tests/otel_tracing_e2e/test_e2e.py +++ b/tests/otel_tracing_e2e/test_e2e.py @@ -1,4 +1,5 @@ -from _validator import validate_trace +import os +from _validator import validate_all_traces from utils import context, weblog, interfaces, scenarios, irrelevant @@ -13,11 +14,41 @@ def test_main(self): otel_trace_ids = set(interfaces.open_telemetry.get_otel_trace_id(request=self.r)) assert len(otel_trace_ids) == 2 dd_trace_ids = [self._get_dd_trace_id(otel_trace_id) for otel_trace_id in otel_trace_ids] - traces = [ - interfaces.backend.assert_otlp_trace_exist(request=self.r, dd_trace_id=dd_trace_id) + + # The 1st account has traces sent by DD Agent + traces_agent = [ + interfaces.backend.assert_otlp_trace_exist( + request=self.r, + dd_trace_id=dd_trace_id, + dd_api_key=os.environ["DD_API_KEY"], + dd_app_key=os.environ.get("DD_APP_KEY", os.environ["DD_APPLICATION_KEY"]), + ) + for dd_trace_id in dd_trace_ids + ] + + # The 2nd account has traces via the backend OTLP intake endpoint + traces_intake = [ + interfaces.backend.assert_otlp_trace_exist( + request=self.r, + dd_trace_id=dd_trace_id, + dd_api_key=os.environ["DD_API_KEY_2"], + dd_app_key=os.environ["DD_APP_KEY_2"], + ) for dd_trace_id in dd_trace_ids ] - validate_trace(traces, self.use_128_bits_trace_id) + + # The 3rd account has traces sent by OTel Collector + traces_collector = [ + interfaces.backend.assert_otlp_trace_exist( + request=self.r, + dd_trace_id=dd_trace_id, + dd_api_key=os.environ["DD_API_KEY_3"], + dd_app_key=os.environ["DD_APP_KEY_3"], + ) + for dd_trace_id in dd_trace_ids + ] + + validate_all_traces(traces_agent, traces_intake, traces_collector, self.use_128_bits_trace_id) def _get_dd_trace_id(self, otel_trace_id=bytes) -> int: if self.use_128_bits_trace_id: diff --git a/utils/_context/_scenarios.py b/utils/_context/_scenarios.py index 07b368ddce..e4827e5130 100644 --- a/utils/_context/_scenarios.py +++ b/utils/_context/_scenarios.py @@ -19,6 +19,7 @@ CassandraContainer, RabbitMqContainer, MySqlContainer, + OpenTelemetryCollectorContainer, create_network, ) @@ -427,12 +428,16 @@ class OpenTelemetryScenario(_DockerScenario): def __init__(self, name, weblog_env) -> None: self._required_containers = [] + self._check_env_vars() super().__init__(name, use_proxy=True) if not self.is_current_scenario: return + self.agent_container = AgentContainer(host_log_folder=self.host_log_folder, use_proxy=True) self.weblog_container = WeblogContainer(self.host_log_folder, environment=weblog_env) + self._required_containers.append(self.agent_container) self._required_containers.append(self.weblog_container) + self._required_containers.append(OpenTelemetryCollectorContainer(self.host_log_folder)) def _create_interface_folders(self): for interface in ("open_telemetry", "backend"): @@ -499,17 +504,18 @@ def _wait_interface(interface, session, timeout): interface.wait(timeout) + def _check_env_vars(self): + for env in ["DD_API_KEY", "DD_APP_KEY", "DD_API_KEY_2", "DD_APP_KEY_2", "DD_API_KEY_3", "DD_APP_KEY_3"]: + if env not in os.environ: + raise Exception(f"Please set {env}, OTel E2E test requires 3 API keys and 3 APP keys") + @property def library(self): return LibraryVersion("open_telemetry", "0.0.0") - @property - def agent(self): - return LibraryVersion("agent", "0.0.0") - @property def agent_version(self): - return self.agent.version + return self.agent_container.agent_version @property def weblog_variant(self): @@ -773,7 +779,7 @@ class scenarios: otel_tracing_e2e = OpenTelemetryScenario( "OTEL_TRACING_E2E", - weblog_env={"DD_API_KEY": os.environ.get("DD_API_KEY"), "DD_SITE": os.environ.get("DD_SITE"),}, + weblog_env={"DD_API_KEY": os.environ.get("DD_API_KEY_2"), "DD_SITE": os.environ.get("DD_SITE", "datad0g.com"),}, ) library_conf_custom_headers_short = EndToEndScenario( diff --git a/utils/_context/containers.py b/utils/_context/containers.py index 4f057055ab..1508d2d737 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -27,7 +27,15 @@ class TestedContainer: # https://docker-py.readthedocs.io/en/stable/containers.html def __init__( - self, name, image_name, host_log_folder, environment=None, allow_old_container=False, healthcheck=None, **kwargs + self, + name, + image_name, + host_log_folder, + environment=None, + allow_old_container=False, + healthcheck=None, + command=None, + **kwargs, ) -> None: self.name = name self.host_log_folder = host_log_folder @@ -41,7 +49,7 @@ def __init__( self.image = ImageInfo(image_name, dir_path=self.log_folder_path) self.healthcheck = healthcheck self.environment = self.image.env | (environment or {}) - + self.command = command self.kwargs = kwargs self._container = None @@ -88,6 +96,7 @@ def start(self) -> Container: name=self.container_name, hostname=self.name, environment=self.environment, + command=self.command, # auto_remove=True, detach=True, network=_NETWORK_NAME, @@ -445,3 +454,20 @@ def __init__(self, host_log_folder) -> None: host_log_folder=host_log_folder, healthcheck={"test": "/healthcheck.sh", "retries": 60}, ) + + +class OpenTelemetryCollectorContainer(TestedContainer): + def __init__(self, host_log_folder) -> None: + super().__init__( + image_name="otel/opentelemetry-collector-contrib:latest", + name="collector", + command="--config=/etc/otelcol-config.yml", + environment={ + "DD_API_KEY": os.environ.get("DD_API_KEY_3"), + "DD_SITE": os.environ.get("DD_SITE", "datad0g.com"), + }, + volumes={"./utils/build/docker/otelcol-config.yaml": {"bind": "/etc/otelcol-config.yml", "mode": "ro",}}, + host_log_folder=host_log_folder, + healthcheck={"test": "curl --fail http://localhost:13133", "retries": 60}, + ports={"13133/tcp": ("0.0.0.0", 13133)}, + ) diff --git a/utils/build/docker/agent.Dockerfile b/utils/build/docker/agent.Dockerfile index 5b9478e6f0..67c9554920 100644 --- a/utils/build/docker/agent.Dockerfile +++ b/utils/build/docker/agent.Dockerfile @@ -11,6 +11,16 @@ RUN echo '\ log_level: DEBUG\n\ apm_config:\n\ apm_non_local_traffic: true\n\ +otlp_config:\n\ + debug:\n\ + verbosity: detailed\n\ + receiver:\n\ + protocols:\n\ + http:\n\ + endpoint: 0.0.0.0:4318\n\ + traces:\n\ + enabled: true\n\ + span_name_as_resource_name: true\n\ ' >> /etc/datadog-agent/datadog.yaml # Proxy conf diff --git a/utils/build/docker/java_otel/spring-boot-native/src/main/java/com/datadoghq/springbootnative/App.java b/utils/build/docker/java_otel/spring-boot-native/src/main/java/com/datadoghq/springbootnative/App.java index 067b8d9df3..5202f8e42b 100644 --- a/utils/build/docker/java_otel/spring-boot-native/src/main/java/com/datadoghq/springbootnative/App.java +++ b/utils/build/docker/java_otel/spring-boot-native/src/main/java/com/datadoghq/springbootnative/App.java @@ -24,15 +24,19 @@ public static void main(String[] args) { ResourceAttributes.SERVICE_NAME, "otel-system-tests-spring-boot", ResourceAttributes.DEPLOYMENT_ENVIRONMENT, "system-tests")); + OtlpHttpSpanExporter agentExporter = + OtlpHttpSpanExporter.builder().setEndpoint("http://proxy:8126/agent/v1/traces").addHeader("dd-protocol", "otlp").build(); OtlpHttpSpanExporter intakeExporter = OtlpHttpSpanExporter.builder() - .setEndpoint("http://proxy:8126/api/v0.2/traces") // send to the proxy first - .addHeader("dd-protocol", "otlp") - .addHeader("dd-api-key", System.getenv("DD_API_KEY")) - .build(); + .setEndpoint("http://proxy:8126/api/v0.2/traces") // send to the proxy first + .addHeader("dd-protocol", "otlp") + .addHeader("dd-api-key", System.getenv("DD_API_KEY")) + .build(); + OtlpHttpSpanExporter collectorExporter = + OtlpHttpSpanExporter.builder().setEndpoint("http://proxy:8126/collector/v1/traces").addHeader("dd-protocol", "otlp").build(); SpanExporter loggingSpanExporter = OtlpJsonLoggingSpanExporter.create(); - SpanExporter exporter = SpanExporter.composite(intakeExporter, loggingSpanExporter); + SpanExporter exporter = SpanExporter.composite(agentExporter, intakeExporter, collectorExporter, loggingSpanExporter); SpanProcessor processor = BatchSpanProcessor.builder(exporter) .setMaxExportBatchSize(1) diff --git a/utils/build/docker/otelcol-config.yaml b/utils/build/docker/otelcol-config.yaml new file mode 100644 index 0000000000..f825e31c6e --- /dev/null +++ b/utils/build/docker/otelcol-config.yaml @@ -0,0 +1,30 @@ +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + +exporters: + datadog: + traces: + span_name_as_resource_name: true + api: + key: ${DD_API_KEY} + site: ${DD_SITE} + fail_on_invalid_key: true + logging: + verbosity: detailed + +processors: + batch: + +extensions: + health_check: + +service: + extensions: [health_check] + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [datadog, logging] \ No newline at end of file diff --git a/utils/interfaces/_backend.py b/utils/interfaces/_backend.py index 2b61b6f0b3..ba654cf780 100644 --- a/utils/interfaces/_backend.py +++ b/utils/interfaces/_backend.py @@ -95,7 +95,9 @@ def assert_library_traces_exist(self, request, min_traces_len=1): ), f"We only found {len(traces)} traces in the library (tracers), but we expected {min_traces_len}!" return traces - def assert_otlp_trace_exist(self, request: requests.Request, dd_trace_id: str) -> dict: + def assert_otlp_trace_exist( + self, request: requests.Request, dd_trace_id: str, dd_api_key: str = None, dd_app_key: str = None + ) -> dict: """Attempts to fetch from the backend, ALL the traces that the OpenTelemetry SDKs sent to Datadog during the execution of the given request. @@ -105,7 +107,14 @@ def assert_otlp_trace_exist(self, request: requests.Request, dd_trace_id: str) - """ rid = get_rid_from_request(request) - data = self._wait_for_trace(rid=rid, trace_id=dd_trace_id, retries=10, sleep_interval_multiplier=2.0) + data = self._wait_for_trace( + rid=rid, + trace_id=dd_trace_id, + retries=10, + sleep_interval_multiplier=2.0, + dd_api_key=dd_api_key, + dd_app_key=dd_app_key, + ) return data["response"]["content"]["trace"] def assert_single_spans_exist(self, request, min_spans_len=1, limit=100): @@ -165,11 +174,14 @@ def _get_trace_ids(self, rid): return self.rid_to_library_trace_ids[rid] - def _request(self, method, path, json_payload=None): - + def _request(self, method, path, json_payload=None, dd_api_key=None, dd_app_key=None): + if dd_api_key is None: + dd_api_key = os.environ["DD_API_KEY"] + if dd_app_key is None: + dd_app_key = os.environ.get("DD_APP_KEY", os.environ["DD_APPLICATION_KEY"]) headers = { - "DD-API-KEY": os.environ["DD_API_KEY"], - "DD-APPLICATION-KEY": os.environ.get("DD_APP_KEY", os.environ["DD_APPLICATION_KEY"]), + "DD-API-KEY": dd_api_key, + "DD-APPLICATION-KEY": dd_app_key, } r = requests.request(method, url=f"{self.dd_site_url}{path}", headers=headers, json=json_payload, timeout=10) @@ -193,21 +205,21 @@ def _request(self, method, path, json_payload=None): return data - def _get_backend_trace_data(self, rid, trace_id): + def _get_backend_trace_data(self, rid, trace_id, dd_api_key=None, dd_app_key=None): path = f"/api/v1/trace/{trace_id}" - result = self._request("GET", path=path) + result = self._request("GET", path=path, dd_api_key=dd_api_key, dd_app_key=dd_app_key) result["rid"] = rid return result - def _wait_for_trace(self, rid, trace_id, retries, sleep_interval_multiplier): + def _wait_for_trace(self, rid, trace_id, retries, sleep_interval_multiplier, dd_api_key=None, dd_app_key=None): sleep_interval_s = 1 current_retry = 1 while current_retry <= retries: logger.info(f"Retry {current_retry}") current_retry += 1 - data = self._get_backend_trace_data(rid, trace_id) + data = self._get_backend_trace_data(rid, trace_id, dd_api_key, dd_app_key) # We should retry fetching from the backend as long as the response is 404. status_code = data["response"]["status_code"] diff --git a/utils/interfaces/_open_telemetry.py b/utils/interfaces/_open_telemetry.py index 4fac92de7d..991e0030dc 100644 --- a/utils/interfaces/_open_telemetry.py +++ b/utils/interfaces/_open_telemetry.py @@ -26,7 +26,7 @@ def ingest_file(self, src_path): return super().ingest_file(src_path) def get_otel_trace_id(self, request): - paths = ["/api/v0.2/traces"] + paths = ["/api/v0.2/traces", "/v1/traces"] rid = get_rid_from_request(request) if rid: @@ -34,7 +34,9 @@ def get_otel_trace_id(self, request): for data in self.get_data(path_filters=paths): export_request = ExportTraceServiceRequest() - content = eval(data["request"]["content"]) # Raw content is a str like "b'\n\x\...'" + raw_content = data["request"]["content"] + # Raw content can be either a str like "b'\n\x\...'" or bytes + content = eval(raw_content) if isinstance(raw_content, str) else raw_content assert export_request.ParseFromString(content) > 0, content for resource_span in export_request.resource_spans: for scope_span in resource_span.scope_spans: diff --git a/utils/proxy/core.py b/utils/proxy/core.py index f94d79f8a8..fd3b0c77e1 100644 --- a/utils/proxy/core.py +++ b/utils/proxy/core.py @@ -107,10 +107,19 @@ def request(self, flow): if flow.request.host in ("proxy", "localhost"): # tracer is the only container that uses the proxy directly - if flow.request.headers.get("dd-protocol") == "otlp": # open telemetry datas - flow.request.host = "trace.agent." + os.environ.get("DD_SITE", "datad0g.com") - flow.request.port = 443 - flow.request.scheme = "https" + if flow.request.headers.get("dd-protocol") == "otlp": + # OTLP ingestion + if "/v1/traces" in flow.request.path: + # OTLP Agent or Collector ingestion + flow.request.host = "agent" if "agent" in flow.request.path else "system-tests-collector" + flow.request.port = 4318 + flow.request.path = "/v1/traces" + flow.request.scheme = "http" + else: + # OTLP backend intake + flow.request.host = "trace.agent." + os.environ.get("DD_SITE", "datad0g.com") + flow.request.port = 443 + flow.request.scheme = "https" else: flow.request.host, flow.request.port = "agent", 8127 flow.request.scheme = "http" From 54b8bee3c06e739dd2fcfa23f1add66302b825a7 Mon Sep 17 00:00:00 2001 From: Yang Song Date: Tue, 25 Apr 2023 13:24:45 -0400 Subject: [PATCH 2/9] Fix CI failure --- .github/workflows/ci.yml | 9 +++++---- utils/_context/_scenarios.py | 2 +- utils/_context/containers.py | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e1d960c7a9..a1f7d6ab5c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -640,11 +640,12 @@ jobs: run: ./run.sh OTEL_TRACING_E2E env: DD_API_KEY: ${{ secrets.DD_API_KEY }} + DD_APPLICATION_KEY: ${{ secrets.DD_APPLICATION_KEY }} DD_APP_KEY: ${{ secrets.DD_APPLICATION_KEY }} - DD_API_KEY_2: ${{ secrets.DD_API_KEY_2 }} - DD_APP_KEY_2: ${{ secrets.DD_APPLICATION_KEY_2 }} - DD_API_KEY_3: ${{ secrets.DD_API_KEY_3 }} - DD_APP_KEY_3: ${{ secrets.DD_APPLICATION_KEY_3 }} + DD_API_KEY_2: ${{ secrets.DD_API_KEY }} + DD_APP_KEY_2: ${{ secrets.DD_APPLICATION_KEY }} + DD_API_KEY_3: ${{ secrets.DD_API_KEY }} + DD_APP_KEY_3: ${{ secrets.DD_APPLICATION_KEY }} - name: Compress logs if: steps.build.outcome == 'success' run: tar -czvf artifact.tar.gz $(ls | grep logs) diff --git a/utils/_context/_scenarios.py b/utils/_context/_scenarios.py index e4827e5130..f1864e3fe3 100644 --- a/utils/_context/_scenarios.py +++ b/utils/_context/_scenarios.py @@ -428,11 +428,11 @@ class OpenTelemetryScenario(_DockerScenario): def __init__(self, name, weblog_env) -> None: self._required_containers = [] - self._check_env_vars() super().__init__(name, use_proxy=True) if not self.is_current_scenario: return + self._check_env_vars() self.agent_container = AgentContainer(host_log_folder=self.host_log_folder, use_proxy=True) self.weblog_container = WeblogContainer(self.host_log_folder, environment=weblog_env) self._required_containers.append(self.agent_container) diff --git a/utils/_context/containers.py b/utils/_context/containers.py index 1508d2d737..e783c4156a 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -468,6 +468,6 @@ def __init__(self, host_log_folder) -> None: }, volumes={"./utils/build/docker/otelcol-config.yaml": {"bind": "/etc/otelcol-config.yml", "mode": "ro",}}, host_log_folder=host_log_folder, - healthcheck={"test": "curl --fail http://localhost:13133", "retries": 60}, + # healthcheck={"test": "curl --fail http://localhost:13133", "retries": 60}, ports={"13133/tcp": ("0.0.0.0", 13133)}, ) From 8f966491f8a4d2b9cf925a29df3b4b6253163a81 Mon Sep 17 00:00:00 2001 From: Yang Song Date: Tue, 25 Apr 2023 17:40:31 -0400 Subject: [PATCH 3/9] Fix health check for otelcol --- utils/_context/_scenarios.py | 2 +- utils/_context/containers.py | 17 ++++++++++++++++- utils/interfaces/_backend.py | 2 +- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/utils/_context/_scenarios.py b/utils/_context/_scenarios.py index f1864e3fe3..756029b1be 100644 --- a/utils/_context/_scenarios.py +++ b/utils/_context/_scenarios.py @@ -440,7 +440,7 @@ def __init__(self, name, weblog_env) -> None: self._required_containers.append(OpenTelemetryCollectorContainer(self.host_log_folder)) def _create_interface_folders(self): - for interface in ("open_telemetry", "backend"): + for interface in ("open_telemetry", "backend", "agent"): self.create_log_subfolder(f"interfaces/{interface}") def _start_interface_watchdog(self): diff --git a/utils/_context/containers.py b/utils/_context/containers.py index e783c4156a..a386573af5 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -5,6 +5,7 @@ import docker from docker.models.containers import Container import pytest +import requests from utils._context.library_version import LibraryVersion, Version from utils.tools import logger @@ -468,6 +469,20 @@ def __init__(self, host_log_folder) -> None: }, volumes={"./utils/build/docker/otelcol-config.yaml": {"bind": "/etc/otelcol-config.yml", "mode": "ro",}}, host_log_folder=host_log_folder, - # healthcheck={"test": "curl --fail http://localhost:13133", "retries": 60}, ports={"13133/tcp": ("0.0.0.0", 13133)}, ) + + # Override wait_for_health because we cannot do docker exec for container opentelemetry-collector-contrib + def wait_for_health(self): + time.sleep(20) # It takes long for otel collector to start + + for i in range(61): + try: + r = requests.get("http://localhost:13133", timeout=1) + logger.debug(f"Healthcheck #{i} on localhost:13133: {r}") + if r.status_code == 200: + return + except Exception as e: + logger.debug(f"Healthcheck #{i} on localhost:13133: {e}") + time.sleep(1) + pytest.exit("localhost:13133 never answered to healthcheck request", 1) diff --git a/utils/interfaces/_backend.py b/utils/interfaces/_backend.py index ba654cf780..31de64801e 100644 --- a/utils/interfaces/_backend.py +++ b/utils/interfaces/_backend.py @@ -110,7 +110,7 @@ def assert_otlp_trace_exist( data = self._wait_for_trace( rid=rid, trace_id=dd_trace_id, - retries=10, + retries=5, sleep_interval_multiplier=2.0, dd_api_key=dd_api_key, dd_app_key=dd_app_key, From 07bfca15465abef1ff3d596fe7319b5354fcd35b Mon Sep 17 00:00:00 2001 From: Yang Song Date: Wed, 26 Apr 2023 12:32:46 -0400 Subject: [PATCH 4/9] Improve OTel proxy and fix CI configs --- .github/workflows/ci.yml | 8 +++---- tests/otel_tracing_e2e/test_e2e.py | 2 +- .../com/datadoghq/springbootnative/App.java | 24 +++++++++++++------ utils/proxy/core.py | 16 ++++++++----- 4 files changed, 32 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a1f7d6ab5c..6e61f4cda8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -642,10 +642,10 @@ jobs: DD_API_KEY: ${{ secrets.DD_API_KEY }} DD_APPLICATION_KEY: ${{ secrets.DD_APPLICATION_KEY }} DD_APP_KEY: ${{ secrets.DD_APPLICATION_KEY }} - DD_API_KEY_2: ${{ secrets.DD_API_KEY }} - DD_APP_KEY_2: ${{ secrets.DD_APPLICATION_KEY }} - DD_API_KEY_3: ${{ secrets.DD_API_KEY }} - DD_APP_KEY_3: ${{ secrets.DD_APPLICATION_KEY }} + DD_API_KEY_2: ${{ secrets.DD_API_KEY_2 }} + DD_APP_KEY_2: ${{ secrets.DD_APP_KEY_2 }} + DD_API_KEY_3: ${{ secrets.DD_API_KEY_3 }} + DD_APP_KEY_3: ${{ secrets.DD_APP_KEY_3 }} - name: Compress logs if: steps.build.outcome == 'success' run: tar -czvf artifact.tar.gz $(ls | grep logs) diff --git a/tests/otel_tracing_e2e/test_e2e.py b/tests/otel_tracing_e2e/test_e2e.py index 0581a73a00..af148e8b40 100644 --- a/tests/otel_tracing_e2e/test_e2e.py +++ b/tests/otel_tracing_e2e/test_e2e.py @@ -21,7 +21,7 @@ def test_main(self): request=self.r, dd_trace_id=dd_trace_id, dd_api_key=os.environ["DD_API_KEY"], - dd_app_key=os.environ.get("DD_APP_KEY", os.environ["DD_APPLICATION_KEY"]), + dd_app_key=os.environ.get("DD_APP_KEY", os.environ.get("DD_APPLICATION_KEY")), ) for dd_trace_id in dd_trace_ids ] diff --git a/utils/build/docker/java_otel/spring-boot-native/src/main/java/com/datadoghq/springbootnative/App.java b/utils/build/docker/java_otel/spring-boot-native/src/main/java/com/datadoghq/springbootnative/App.java index 5202f8e42b..7bd6ea5efe 100644 --- a/utils/build/docker/java_otel/spring-boot-native/src/main/java/com/datadoghq/springbootnative/App.java +++ b/utils/build/docker/java_otel/spring-boot-native/src/main/java/com/datadoghq/springbootnative/App.java @@ -25,14 +25,24 @@ public static void main(String[] args) { ResourceAttributes.DEPLOYMENT_ENVIRONMENT, "system-tests")); OtlpHttpSpanExporter agentExporter = - OtlpHttpSpanExporter.builder().setEndpoint("http://proxy:8126/agent/v1/traces").addHeader("dd-protocol", "otlp").build(); - OtlpHttpSpanExporter intakeExporter = OtlpHttpSpanExporter.builder() - .setEndpoint("http://proxy:8126/api/v0.2/traces") // send to the proxy first - .addHeader("dd-protocol", "otlp") - .addHeader("dd-api-key", System.getenv("DD_API_KEY")) - .build(); + OtlpHttpSpanExporter.builder() + .setEndpoint("http://proxy:8126/v1/traces") + .addHeader("dd-protocol", "otlp") + .addHeader("dd-otlp-path", "agent") + .build(); + OtlpHttpSpanExporter intakeExporter = + OtlpHttpSpanExporter.builder() + .setEndpoint("http://proxy:8126/api/v0.2/traces") // send to the proxy first + .addHeader("dd-protocol", "otlp") + .addHeader("dd-api-key", System.getenv("DD_API_KEY")) + .addHeader("dd-otlp-path", "intake") + .build(); OtlpHttpSpanExporter collectorExporter = - OtlpHttpSpanExporter.builder().setEndpoint("http://proxy:8126/collector/v1/traces").addHeader("dd-protocol", "otlp").build(); + OtlpHttpSpanExporter.builder() + .setEndpoint("http://proxy:8126/v1/traces") + .addHeader("dd-protocol", "otlp") + .addHeader("dd-otlp-path", "collector") + .build(); SpanExporter loggingSpanExporter = OtlpJsonLoggingSpanExporter.create(); diff --git a/utils/proxy/core.py b/utils/proxy/core.py index fd3b0c77e1..0edebd6827 100644 --- a/utils/proxy/core.py +++ b/utils/proxy/core.py @@ -109,17 +109,21 @@ def request(self, flow): if flow.request.headers.get("dd-protocol") == "otlp": # OTLP ingestion - if "/v1/traces" in flow.request.path: - # OTLP Agent or Collector ingestion - flow.request.host = "agent" if "agent" in flow.request.path else "system-tests-collector" + otlp_path = flow.request.headers.get("dd-otlp-path") + if otlp_path == "agent": + flow.request.host = "agent" flow.request.port = 4318 - flow.request.path = "/v1/traces" flow.request.scheme = "http" - else: - # OTLP backend intake + elif otlp_path == "collector": + flow.request.host = "system-tests-collector" + flow.request.port = 4318 + flow.request.scheme = "http" + elif otlp_path == "intake": flow.request.host = "trace.agent." + os.environ.get("DD_SITE", "datad0g.com") flow.request.port = 443 flow.request.scheme = "https" + else: + raise Exception(f"Unknown OTLP ingestion path {otlp_path}") else: flow.request.host, flow.request.port = "agent", 8127 flow.request.scheme = "http" From c180db9ecdb42d510140ede368bf6d6d57ece82f Mon Sep 17 00:00:00 2001 From: Yang Song Date: Thu, 27 Apr 2023 11:12:13 -0400 Subject: [PATCH 5/9] Fix CI failure --- utils/_context/_scenarios.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/_context/_scenarios.py b/utils/_context/_scenarios.py index d33b604b7a..146e776e3b 100644 --- a/utils/_context/_scenarios.py +++ b/utils/_context/_scenarios.py @@ -568,6 +568,9 @@ def _wait_interface(interface, session, timeout): interface.wait(timeout) def _check_env_vars(self): + if os.environ.get("SYSTEMTESTS_SCENARIO", "EMPTY_SCENARIO") != self.name: + return + for env in ["DD_API_KEY", "DD_APP_KEY", "DD_API_KEY_2", "DD_APP_KEY_2", "DD_API_KEY_3", "DD_APP_KEY_3"]: if env not in os.environ: raise Exception(f"Please set {env}, OTel E2E test requires 3 API keys and 3 APP keys") From c11ba80d79024d71031187d828400304514dc646 Mon Sep 17 00:00:00 2001 From: Yang Song Date: Thu, 27 Apr 2023 12:48:06 -0400 Subject: [PATCH 6/9] Improve env var check --- utils/_context/_scenarios.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/_context/_scenarios.py b/utils/_context/_scenarios.py index 146e776e3b..909f827cff 100644 --- a/utils/_context/_scenarios.py +++ b/utils/_context/_scenarios.py @@ -572,7 +572,7 @@ def _check_env_vars(self): return for env in ["DD_API_KEY", "DD_APP_KEY", "DD_API_KEY_2", "DD_APP_KEY_2", "DD_API_KEY_3", "DD_APP_KEY_3"]: - if env not in os.environ: + if os.environ.get(env, "") == "": raise Exception(f"Please set {env}, OTel E2E test requires 3 API keys and 3 APP keys") @property From 2ddb254f6b04ebdd412d0949fe7ecc1b2559a55c Mon Sep 17 00:00:00 2001 From: Yang Song Date: Thu, 27 Apr 2023 13:08:43 -0400 Subject: [PATCH 7/9] Increase retries in get trace --- utils/interfaces/_backend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/interfaces/_backend.py b/utils/interfaces/_backend.py index aca8b592ee..9ec20deca3 100644 --- a/utils/interfaces/_backend.py +++ b/utils/interfaces/_backend.py @@ -110,7 +110,7 @@ def assert_otlp_trace_exist( data = self._wait_for_trace( rid=rid, trace_id=dd_trace_id, - retries=5, + retries=8, sleep_interval_multiplier=2.0, dd_api_key=dd_api_key, dd_app_key=dd_app_key, From d8943db5aa7dfb0c921ec057dd05b7a772103937 Mon Sep 17 00:00:00 2001 From: Yang Song Date: Tue, 2 May 2023 16:05:59 -0400 Subject: [PATCH 8/9] Fix review comments --- tests/otel_tracing_e2e/test_e2e.py | 8 +++++--- utils/_context/_scenarios.py | 28 +++++++++++++++------------- utils/_context/containers.py | 5 +---- utils/interfaces/_deserializer.py | 15 +++++++++++++++ utils/interfaces/_open_telemetry.py | 24 ++++++++---------------- 5 files changed, 44 insertions(+), 36 deletions(-) diff --git a/tests/otel_tracing_e2e/test_e2e.py b/tests/otel_tracing_e2e/test_e2e.py index af148e8b40..3a7373de87 100644 --- a/tests/otel_tracing_e2e/test_e2e.py +++ b/tests/otel_tracing_e2e/test_e2e.py @@ -1,3 +1,4 @@ +import base64 import os from _validator import validate_all_traces from utils import context, weblog, interfaces, scenarios, irrelevant @@ -50,7 +51,8 @@ def test_main(self): validate_all_traces(traces_agent, traces_intake, traces_collector, self.use_128_bits_trace_id) - def _get_dd_trace_id(self, otel_trace_id=bytes) -> int: + def _get_dd_trace_id(self, otel_trace_id=str) -> int: + otel_trace_id_bytes = base64.b64decode(otel_trace_id) if self.use_128_bits_trace_id: - return int.from_bytes(otel_trace_id, "big") - return int.from_bytes(otel_trace_id[8:], "big") + return int.from_bytes(otel_trace_id_bytes, "big") + return int.from_bytes(otel_trace_id_bytes[8:], "big") diff --git a/utils/_context/_scenarios.py b/utils/_context/_scenarios.py index 3c3d94ded1..003243c54d 100644 --- a/utils/_context/_scenarios.py +++ b/utils/_context/_scenarios.py @@ -494,16 +494,24 @@ def get_junit_properties(self): class OpenTelemetryScenario(_DockerScenario): """ Scenario for testing opentelemetry""" - def __init__(self, name, weblog_env) -> None: - self._required_containers = [] + def __init__(self, name) -> None: super().__init__(name, use_proxy=True) - self._check_env_vars() self.agent_container = AgentContainer(host_log_folder=self.host_log_folder, use_proxy=True) - self.weblog_container = WeblogContainer(self.host_log_folder, environment=weblog_env) + self.weblog_container = WeblogContainer(self.host_log_folder) + self.collector_container = OpenTelemetryCollectorContainer(self.host_log_folder) self._required_containers.append(self.agent_container) self._required_containers.append(self.weblog_container) - self._required_containers.append(OpenTelemetryCollectorContainer(self.host_log_folder)) + self._required_containers.append(self.collector_container) + + def configure(self): + super().configure() + self._check_env_vars() + dd_site = os.environ.get("DD_SITE", "datad0g.com") + self.weblog_container.environment["DD_API_KEY"] = os.environ.get("DD_API_KEY_2") + self.weblog_container.environment["DD_SITE"] = dd_site + self.collector_container.environment["DD_API_KEY"] = os.environ.get("DD_API_KEY_3") + self.collector_container.environment["DD_SITE"] = dd_site def _create_interface_folders(self): for interface in ("open_telemetry", "backend", "agent"): @@ -571,11 +579,8 @@ def _wait_interface(interface, session, timeout): interface.wait(timeout) def _check_env_vars(self): - if os.environ.get("SYSTEMTESTS_SCENARIO", "EMPTY_SCENARIO") != self.name: - return - for env in ["DD_API_KEY", "DD_APP_KEY", "DD_API_KEY_2", "DD_APP_KEY_2", "DD_API_KEY_3", "DD_APP_KEY_3"]: - if os.environ.get(env, "") == "": + if env not in os.environ: raise Exception(f"Please set {env}, OTel E2E test requires 3 API keys and 3 APP keys") @property @@ -844,10 +849,7 @@ class scenarios: backend_interface_timeout=5, ) - otel_tracing_e2e = OpenTelemetryScenario( - "OTEL_TRACING_E2E", - weblog_env={"DD_API_KEY": os.environ.get("DD_API_KEY_2"), "DD_SITE": os.environ.get("DD_SITE", "datad0g.com"),}, - ) + otel_tracing_e2e = OpenTelemetryScenario("OTEL_TRACING_E2E") library_conf_custom_headers_short = EndToEndScenario( "LIBRARY_CONF_CUSTOM_HEADERS_SHORT", additional_trace_header_tags=("header-tag1", "header-tag2") diff --git a/utils/_context/containers.py b/utils/_context/containers.py index e1d492daf3..c1850886dd 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -489,10 +489,7 @@ def __init__(self, host_log_folder) -> None: image_name="otel/opentelemetry-collector-contrib:latest", name="collector", command="--config=/etc/otelcol-config.yml", - environment={ - "DD_API_KEY": os.environ.get("DD_API_KEY_3"), - "DD_SITE": os.environ.get("DD_SITE", "datad0g.com"), - }, + environment={}, volumes={"./utils/build/docker/otelcol-config.yaml": {"bind": "/etc/otelcol-config.yml", "mode": "ro",}}, host_log_folder=host_log_folder, ports={"13133/tcp": ("0.0.0.0", 13133)}, diff --git a/utils/interfaces/_deserializer.py b/utils/interfaces/_deserializer.py index 9255fc585a..c9b8660543 100644 --- a/utils/interfaces/_deserializer.py +++ b/utils/interfaces/_deserializer.py @@ -7,6 +7,10 @@ import msgpack from requests_toolbelt.multipart.decoder import MultipartDecoder from google.protobuf.json_format import MessageToDict +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, + ExportTraceServiceResponse, +) from utils.interfaces._decoders.protobuf_schemas import TracePayload from utils.tools import logger @@ -106,6 +110,17 @@ def json_load(): return result + dd_protocol = get_header_value("dd-protocol", message["headers"]) + if content_type == "application/x-protobuf" and dd_protocol == "otlp": + # Raw data can be either a str like "b'\n\x\...'" or bytes + content = eval(data) if isinstance(data, str) else data + return MessageToDict(ExportTraceServiceRequest.FromString(content)) + + if content_type == "application/x-protobuf" and path == "/v1/traces": + # Raw data can be either a str like "b'\n\x\...'" or bytes + content = eval(data) if isinstance(data, str) else data + return MessageToDict(ExportTraceServiceResponse.FromString(content)) + if content_type == "application/x-protobuf" and path == "/api/v0.2/traces": return MessageToDict(TracePayload.FromString(data)) diff --git a/utils/interfaces/_open_telemetry.py b/utils/interfaces/_open_telemetry.py index 991e0030dc..78dbc7bd8d 100644 --- a/utils/interfaces/_open_telemetry.py +++ b/utils/interfaces/_open_telemetry.py @@ -8,8 +8,6 @@ import threading -from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest - from utils.tools import logger from utils.interfaces._core import InterfaceValidator, get_rid_from_request @@ -33,17 +31,11 @@ def get_otel_trace_id(self, request): logger.debug(f"Try to find traces related to request {rid}") for data in self.get_data(path_filters=paths): - export_request = ExportTraceServiceRequest() - raw_content = data["request"]["content"] - # Raw content can be either a str like "b'\n\x\...'" or bytes - content = eval(raw_content) if isinstance(raw_content, str) else raw_content - assert export_request.ParseFromString(content) > 0, content - for resource_span in export_request.resource_spans: - for scope_span in resource_span.scope_spans: - for span in scope_span.spans: - for attribute in span.attributes: - if ( - attribute.key == "http.request.headers.user-agent" - and rid in attribute.value.string_value - ): - yield span.trace_id + for resource_span in data.get("request").get("content").get("resourceSpans"): + for scope_span in resource_span.get("scopeSpans"): + for span in scope_span.get("spans"): + for attribute in span.get("attributes"): + attr_key = attribute.get("key") + attr_val = attribute.get("value").get("stringValue") + if attr_key == "http.request.headers.user-agent" and rid in attr_val: + yield span.get("traceId") From 9efa94920c80bbae370434ab42de584636edb901 Mon Sep 17 00:00:00 2001 From: Yang Song Date: Thu, 4 May 2023 12:01:44 -0400 Subject: [PATCH 9/9] Separate healthcheck and main test scenario --- tests/otel_tracing_e2e/_validator.py | 4 ++-- tests/otel_tracing_e2e/test_e2e.py | 2 +- .../datadoghq/springbootnative/WebController.java | 14 +++++++++++--- utils/interfaces/_open_telemetry.py | 2 +- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/tests/otel_tracing_e2e/_validator.py b/tests/otel_tracing_e2e/_validator.py index 365134122e..4512bd0440 100644 --- a/tests/otel_tracing_e2e/_validator.py +++ b/tests/otel_tracing_e2e/_validator.py @@ -65,14 +65,14 @@ def validate_trace_id(span: dict, use_128_bits_trace_id: bool): def validate_server_span(span: dict): - expected_tags = {"name": "WebController.home", "resource": "GET /"} + expected_tags = {"name": "WebController.basic", "resource": "GET /"} expected_meta = {"http.route": "/", "http.method": "GET"} assert expected_tags.items() <= span.items() assert expected_meta.items() <= span["meta"].items() def validate_message_span(span: dict): - expected_tags = {"name": "WebController.home.publish", "resource": "publish"} + expected_tags = {"name": "WebController.basic.publish", "resource": "publish"} expected_meta = {"messaging.operation": "publish", "messaging.system": "rabbitmq"} assert expected_tags.items() <= span.items() assert expected_meta.items() <= span["meta"].items() diff --git a/tests/otel_tracing_e2e/test_e2e.py b/tests/otel_tracing_e2e/test_e2e.py index 3a7373de87..7218dae116 100644 --- a/tests/otel_tracing_e2e/test_e2e.py +++ b/tests/otel_tracing_e2e/test_e2e.py @@ -9,7 +9,7 @@ class Test_OTel_E2E: def setup_main(self): self.use_128_bits_trace_id = False - self.r = weblog.get(path="/") + self.r = weblog.get(path="/basic") def test_main(self): otel_trace_ids = set(interfaces.open_telemetry.get_otel_trace_id(request=self.r)) diff --git a/utils/build/docker/java_otel/spring-boot-native/src/main/java/com/datadoghq/springbootnative/WebController.java b/utils/build/docker/java_otel/spring-boot-native/src/main/java/com/datadoghq/springbootnative/WebController.java index c73c4e898b..12a0eac486 100644 --- a/utils/build/docker/java_otel/spring-boot-native/src/main/java/com/datadoghq/springbootnative/WebController.java +++ b/utils/build/docker/java_otel/spring-boot-native/src/main/java/com/datadoghq/springbootnative/WebController.java @@ -19,11 +19,19 @@ public class WebController { private final Tracer tracer = GlobalOpenTelemetry.getTracer("com.datadoghq.springbootnative"); + // Home '/' is only used for health check, it generates and sends one span to proxy to indicate interfaces are ready. @RequestMapping("/") - private String home(@RequestHeader HttpHeaders headers) throws InterruptedException { + private String home(@RequestHeader HttpHeaders headers) { + tracer.spanBuilder("Healthcheck").setSpanKind(SpanKind.SERVER).startSpan().end(); + return "Weblog is ready"; + } + + // Basic test scenario that generates a server span with a span link to a fake message span. + @RequestMapping("/basic") + private String basic(@RequestHeader HttpHeaders headers) throws InterruptedException { try (Scope scope = Context.current().makeCurrent()) { SpanContext spanContext = fakeAsyncWork(headers); - Span span = tracer.spanBuilder("WebController.home") + Span span = tracer.spanBuilder("WebController.basic") .setSpanKind(SpanKind.SERVER) .addLink(spanContext, Attributes.of(AttributeKey.stringKey("messaging.operation"), "publish")) .setAttribute(SemanticAttributes.HTTP_ROUTE, "/") @@ -41,7 +49,7 @@ private String home(@RequestHeader HttpHeaders headers) throws InterruptedExcept // Create a fake producer span and return its span context to test span links private SpanContext fakeAsyncWork(HttpHeaders headers) throws InterruptedException { - Span fakeSpan = tracer.spanBuilder("WebController.home.publish") + Span fakeSpan = tracer.spanBuilder("WebController.basic.publish") .setSpanKind(SpanKind.PRODUCER) .setAttribute("messaging.system", "rabbitmq") .setAttribute("messaging.operation", "publish") diff --git a/utils/interfaces/_open_telemetry.py b/utils/interfaces/_open_telemetry.py index 78dbc7bd8d..125443f8f5 100644 --- a/utils/interfaces/_open_telemetry.py +++ b/utils/interfaces/_open_telemetry.py @@ -34,7 +34,7 @@ def get_otel_trace_id(self, request): for resource_span in data.get("request").get("content").get("resourceSpans"): for scope_span in resource_span.get("scopeSpans"): for span in scope_span.get("spans"): - for attribute in span.get("attributes"): + for attribute in span.get("attributes", []): attr_key = attribute.get("key") attr_val = attribute.get("value").get("stringValue") if attr_key == "http.request.headers.user-agent" and rid in attr_val: