Capabilities tests should be stricter

tests/parametric/conftest.py

@@ -1,5 +1,5 @@
 import base64
-from collections.abc import Iterable, Generator
+from collections.abc import Generator
 import contextlib
 import dataclasses
 import os
@@ -23,7 +23,7 @@
 from utils.parametric._library_client import APMLibrary, APMLibraryClient
 
 from utils import context, scenarios
-from utils.dd_constants import RemoteConfigApplyState
+from utils.dd_constants import RemoteConfigApplyState, Capabilities
 from utils.tools import logger
 
 from utils._context._scenarios.parametric import APMLibraryTestServer
@@ -112,13 +112,13 @@ def _write_log(self, log_type, json_trace):
             log.write(f"\n{log_type}>>>>\n")
             log.write(json.dumps(json_trace))
 
-    def traces(self, clear=False, **kwargs):
+    def traces(self, clear=False, **kwargs) -> list[Trace]:
         resp = self._session.get(self._url("/test/session/traces"), **kwargs)
         if clear:
             self.clear()
-        json = resp.json()
-        self._write_log("traces", json)
-        return json
+        resp_json = resp.json()
+        self._write_log("traces", resp_json)
+        return resp_json
 
     def set_remote_config(self, path, payload):
         resp = self._session.post(self._url("/test/session/responses/config/path"), json={"path": path, "msg": payload})
@@ -214,7 +214,7 @@ def set_trace_delay(self, delay):
         resp = self._session.post(self._url("/test/settings"), json={"trace_request_delay": delay})
         assert resp.status_code == 202
 
-    def raw_telemetry(self, clear=False, **kwargs):
+    def raw_telemetry(self, clear=False):
         raw_reqs = self.requests()
         reqs = []
         for req in raw_reqs:
@@ -232,42 +232,42 @@ def telemetry(self, clear=False, **kwargs):
 
     def tracestats(self, **kwargs):
         resp = self._session.get(self._url("/test/session/stats"), **kwargs)
-        json = resp.json()
-        self._write_log("tracestats", json)
-        return json
+        resp_json = resp.json()
+        self._write_log("tracestats", resp_json)
+        return resp_json
 
     def requests(self, **kwargs) -> list[AgentRequest]:
         resp = self._session.get(self._url("/test/session/requests"), **kwargs)
-        json = resp.json()
-        self._write_log("requests", json)
-        return json
+        resp_json = resp.json()
+        self._write_log("requests", resp_json)
+        return resp_json
 
     def rc_requests(self, post_only=False):
         reqs = self.requests()
-        rc_reqs = [r for r in reqs if r["url"].endswith("/v0.7/config") and (not (post_only) or r["method"] == "POST")]
+        rc_reqs = [r for r in reqs if r["url"].endswith("/v0.7/config") and (not post_only or r["method"] == "POST")]
         for r in rc_reqs:
             r["body"] = json.loads(base64.b64decode(r["body"]).decode("utf-8"))
         return rc_reqs
 
     def get_tracer_flares(self, **kwargs):
         resp = self._session.get(self._url("/test/session/tracerflares"), **kwargs)
-        json = resp.json()
-        self._write_log("tracerflares", json)
-        return json
+        resp_json = resp.json()
+        self._write_log("tracerflares", resp_json)
+        return resp_json
 
     def v06_stats_requests(self) -> list[AgentRequestV06Stats]:
         raw_requests = [r for r in self.requests() if "/v0.6/stats" in r["url"]]
-        requests = []
+        agent_requests = []
         for raw in raw_requests:
-            requests.append(
+            agent_requests.append(
                 AgentRequestV06Stats(
                     method=raw["method"],
                     url=raw["url"],
                     headers=raw["headers"],
                     body=decode_v06_stats(base64.b64decode(raw["body"])),
                 )
             )
-        return requests
+        return agent_requests
 
     def clear(self, **kwargs) -> None:
         self._session.get(self._url("/test/session/clear"), **kwargs)
@@ -280,9 +280,9 @@ def info(self, **kwargs):
             logger.error(message)
             raise ValueError(message)
 
-        json = resp.json()
-        self._write_log("info", json)
-        return json
+        resp_json = resp.json()
+        self._write_log("info", resp_json)
+        return resp_json
 
     @contextlib.contextmanager
     def snapshot_context(self, token, ignores=None):
@@ -313,6 +313,7 @@ def wait_for_num_traces(
         When sort_by_start=True returned traces are sorted by the span start time to simplify assertions by knowing that returned traces are in the same order as they have been created.
         """
         num_received = None
+        traces = []
         for i in range(wait_loops):
             try:
                 traces = self.traces(clear=False)
@@ -328,7 +329,7 @@ def wait_for_num_traces(
                             # The testagent may receive spans and trace chunks in any order,
                             # so we sort the spans by start time if needed
                             trace.sort(key=lambda x: x["start"])
-                        return sorted(traces, key=lambda trace: trace[0]["start"])
+                        return sorted(traces, key=lambda t: t[0]["start"])
                     return traces
             time.sleep(0.1)
         raise ValueError(f"Number ({num}) of traces not available from test agent, got {num_received}:\n{traces}")
@@ -361,7 +362,7 @@ def wait_for_num_spans(
                             # The testagent may receive spans and trace chunks in any order,
                             # so we sort the spans by start time if needed
                             trace.sort(key=lambda x: x["start"])
-                        return sorted(traces, key=lambda trace: trace[0]["start"])
+                        return sorted(traces, key=lambda t: t[0]["start"])
                     return traces
             time.sleep(0.1)
         raise ValueError(f"Number ({num}) of spans not available from test agent, got {num_received}")
@@ -420,7 +421,7 @@ def wait_for_rc_apply_state(
                             logger.debug(f"Product {cfg_state['product']} does not match {product}")
                         elif cfg_state["apply_state"] != state.value:
                             if last_known_state != cfg_state["apply_state"]:
-                                # this condition prevent to spam logs, because the last knwon state
+                                # this condition prevent to spam logs, because the last known state
                                 # will probably be the same as the current state
                                 last_known_state = cfg_state["apply_state"]
                                 logger.debug(f"Apply state {cfg_state['apply_state']} does not match {state}")
@@ -432,10 +433,8 @@ def wait_for_rc_apply_state(
             time.sleep(0.01)
         raise AssertionError(f"No RemoteConfig apply status found, got requests {rc_reqs}")
 
-    def wait_for_rc_capabilities(self, capabilities: Iterable[int] = (), wait_loops: int = 100):
+    def wait_for_rc_capabilities(self, wait_loops: int = 100) -> set[Capabilities]:
         """Wait for the given RemoteConfig apply state to be received by the test agent."""
-        rc_reqs = []
-        capabilities_seen = set()
         for i in range(wait_loops):
             try:
                 rc_reqs = self.rc_requests()
@@ -456,12 +455,32 @@ def wait_for_rc_capabilities(self, capabilities: Iterable[int] = (), wait_loops:
                         # base64-encoded string:
                         else:
                             decoded_capabilities = base64.b64decode(raw_caps)
+
                         int_capabilities = int.from_bytes(decoded_capabilities, byteorder="big")
-                        capabilities_seen.add(remoteconfig.human_readable_capabilities(int_capabilities))
-                        if all((int_capabilities >> c) & 1 for c in capabilities):
-                            return int_capabilities
+
+                        if int_capabilities >= (1 << 64):
+                            raise AssertionError(
+                                f"RemoteConfig capabilities should only use 64 bits, {int_capabilities}"
+                            )
+
+                        valid_bits = sum(1 << c for c in Capabilities)
+                        if int_capabilities & ~valid_bits != 0:
+                            raise AssertionError(
+                                f"RemoteConfig capabilities contains unknown bits: {bin(int_capabilities & ~valid_bits)}"
+                            )
+
+                        capabilities_seen = remoteconfig.human_readable_capabilities(int_capabilities)
+                        if len(capabilities_seen) > 0:
+                            return capabilities_seen
             time.sleep(0.01)
-        raise AssertionError(f"No RemoteConfig capabilities found, got capabilites {capabilities_seen}")
+        raise AssertionError("RemoteConfig capabilities were empty")
+
+    def assert_rc_capabilities(self, expected_capabilities: set[Capabilities], wait_loops: int = 100) -> None:
+        """Wait for the given RemoteConfig apply state to be received by the test agent."""
+        seen_capabilities = self.wait_for_rc_capabilities(wait_loops)
+        missing_capabilities = expected_capabilities.difference(seen_capabilities)
+        if missing_capabilities:
+            raise AssertionError(f"RemoteConfig capabilities missing: {missing_capabilities}")
 
     def wait_for_tracer_flare(self, case_id: str | None = None, clear: bool = False, wait_loops: int = 100):
         """Wait for the tracer-flare to be received by the test agent."""
@@ -511,7 +530,7 @@ def docker_network(test_id: str) -> Generator[str, None, None]:
             network.remove()
         except:
             # It's possible (why?) of having some container not stopped.
-            # If it happen, failing here makes stdout tough to understance.
+            # If it happens, failing here makes stdout tough to understand.
             # Let's ignore this, later calls will clean the mess
             logger.info("Failed to remove network, ignoring the error")
 
@@ -585,7 +604,7 @@ def test_agent(
         network=docker_network,
     ):
         client = _TestAgentAPI(base_url=f"http://localhost:{host_port}", pytest_request=request)
-        time.sleep(0.2)  # intial wait time, the trace agent takes 200ms to start
+        time.sleep(0.2)  # initial wait time, the trace agent takes 200ms to start
         for _ in range(100):
             try:
                 resp = client.info()

tests/parametric/test_dynamic_configuration.py

@@ -125,14 +125,108 @@ def get_sampled_trace(test_library, test_agent, service, name, tags=None):
 
 ENV_SAMPLING_RULE_RATE = 0.55
 
+DEFAULT_SUPPORTED_CAPABILITIES_BY_LANG: dict[str, set[Capabilities]] = {
+    "java": {
+        Capabilities.ASM_ACTIVATION,
+        Capabilities.ASM_IP_BLOCKING,
+        Capabilities.ASM_DD_RULES,
+        Capabilities.ASM_EXCLUSIONS,
+        Capabilities.ASM_REQUEST_BLOCKING,
+        Capabilities.ASM_USER_BLOCKING,
+        Capabilities.ASM_CUSTOM_RULES,
+        Capabilities.ASM_CUSTOM_BLOCKING_RESPONSE,
+        Capabilities.ASM_TRUSTED_IPS,
+        Capabilities.ASM_API_SECURITY_SAMPLE_RATE,
+        Capabilities.APM_TRACING_SAMPLE_RATE,
+        Capabilities.APM_TRACING_LOGS_INJECTION,
+        Capabilities.APM_TRACING_HTTP_HEADER_TAGS,
+        Capabilities.APM_TRACING_CUSTOM_TAGS,
+        Capabilities.ASM_EXCLUSION_DATA,
+        Capabilities.APM_TRACING_ENABLED,
+        Capabilities.APM_TRACING_DATA_STREAMS_ENABLED,
+        Capabilities.ASM_RASP_SQLI,
+        Capabilities.ASM_RASP_LFI,
+        Capabilities.ASM_RASP_SSRF,
+        Capabilities.ASM_RASP_SHI,
+        Capabilities.APM_TRACING_SAMPLE_RULES,
+        Capabilities.ASM_AUTO_USER_INSTRUM_MODE,
+        Capabilities.ASM_ENDPOINT_FINGERPRINT,
+        Capabilities.ASM_SESSION_FINGERPRINT,
+        Capabilities.ASM_NETWORK_FINGERPRINT,
+        Capabilities.ASM_HEADER_FINGERPRINT,
+        Capabilities.ASM_RASP_CMDI,
+    },
+    "nodejs": {
+        Capabilities.ASM_ACTIVATION,
+        Capabilities.APM_TRACING_SAMPLE_RATE,
+        Capabilities.APM_TRACING_LOGS_INJECTION,
+        Capabilities.APM_TRACING_HTTP_HEADER_TAGS,
+        Capabilities.APM_TRACING_CUSTOM_TAGS,
+        Capabilities.APM_TRACING_ENABLED,
+        Capabilities.APM_TRACING_SAMPLE_RULES,
+        Capabilities.ASM_AUTO_USER_INSTRUM_MODE,
+    },
+    "python": {Capabilities.APM_TRACING_ENABLED},
+    "dotnet": {
+        Capabilities.ASM_ACTIVATION,
+        Capabilities.ASM_IP_BLOCKING,
+        Capabilities.ASM_DD_RULES,
+        Capabilities.ASM_EXCLUSIONS,
+        Capabilities.ASM_REQUEST_BLOCKING,
+        Capabilities.ASM_ASM_RESPONSE_BLOCKING,
+        Capabilities.ASM_USER_BLOCKING,
+        Capabilities.ASM_CUSTOM_RULES,
+        Capabilities.ASM_CUSTOM_BLOCKING_RESPONSE,
+        Capabilities.ASM_TRUSTED_IPS,
+        Capabilities.APM_TRACING_SAMPLE_RATE,
+        Capabilities.APM_TRACING_LOGS_INJECTION,
+        Capabilities.APM_TRACING_HTTP_HEADER_TAGS,
+        Capabilities.APM_TRACING_CUSTOM_TAGS,
+        Capabilities.APM_TRACING_ENABLED,
+        Capabilities.APM_TRACING_SAMPLE_RULES,
+        Capabilities.ASM_AUTO_USER_INSTRUM_MODE,
+    },
+    "cpp": {
+        Capabilities.APM_TRACING_SAMPLE_RATE,
+        Capabilities.APM_TRACING_SAMPLE_RULES,
+        Capabilities.APM_TRACING_CUSTOM_TAGS,
+        Capabilities.APM_TRACING_ENABLED,
+    },
+    "php": {Capabilities.APM_TRACING_ENABLED},
+    "golang": {
+        Capabilities.ASM_ACTIVATION,
+        Capabilities.APM_TRACING_SAMPLE_RATE,
+        Capabilities.APM_TRACING_HTTP_HEADER_TAGS,
+        Capabilities.APM_TRACING_CUSTOM_TAGS,
+        Capabilities.APM_TRACING_ENABLED,
+        Capabilities.APM_TRACING_SAMPLE_RULES,
+    },
+    "ruby": {Capabilities.APM_TRACING_ENABLED},
+}
+
 
 @scenarios.parametric
 @features.dynamic_configuration
 class TestDynamicConfigTracingEnabled:
+    @parametrize("library_env", [{**DEFAULT_ENVVARS}])
+    def test_default_capability_completeness(self, library_env, test_agent, test_library):
+        """Ensure the RC request contains the expected default capabilities per language, no more and no less."""
+        if context.library is not None and context.library.library is not None:
+            seen_capabilities = test_agent.wait_for_rc_capabilities()
+            expected_capabilities = DEFAULT_SUPPORTED_CAPABILITIES_BY_LANG[context.library.library]
+
+            seen_but_not_expected_capabilities = seen_capabilities.difference(expected_capabilities)
+            expected_but_not_seen_capabilities = expected_capabilities.difference(seen_capabilities)
+
+            if seen_but_not_expected_capabilities or expected_but_not_seen_capabilities:
+                raise AssertionError(
+                    f"seen_but_not_expected_capabilities={seen_but_not_expected_capabilities}; expected_but_not_seen_capabilities={expected_but_not_seen_capabilities}"
+                )
+
     @parametrize("library_env", [{**DEFAULT_ENVVARS}])
     def test_capability_tracing_enabled(self, library_env, test_agent, test_library):
         """Ensure the RC request contains the tracing enabled capability."""
-        test_agent.wait_for_rc_capabilities([Capabilities.APM_TRACING_ENABLED])
+        test_agent.assert_rc_capabilities({Capabilities.APM_TRACING_ENABLED})
 
     @parametrize("library_env", [{**DEFAULT_ENVVARS}, {**DEFAULT_ENVVARS, "DD_TRACE_ENABLED": "false"}])
     def test_tracing_client_tracing_enabled(self, library_env, test_agent, test_library):
@@ -501,24 +595,24 @@ def test_tracing_client_tracing_tags(self, library_env, test_agent, test_library
     @parametrize("library_env", [{**DEFAULT_ENVVARS}])
     def test_capability_tracing_sample_rate(self, library_env, test_agent, test_library):
         """Ensure the RC request contains the trace sampling rate capability."""
-        test_agent.wait_for_rc_capabilities([Capabilities.APM_TRACING_SAMPLE_RATE])
+        test_agent.assert_rc_capabilities({Capabilities.APM_TRACING_SAMPLE_RATE})
 
     @irrelevant(context.library in ("cpp", "golang"), reason="Tracer doesn't support automatic logs injection")
     @parametrize("library_env", [{**DEFAULT_ENVVARS}])
     def test_capability_tracing_logs_injection(self, library_env, test_agent, test_library):
         """Ensure the RC request contains the logs injection capability."""
-        test_agent.wait_for_rc_capabilities([Capabilities.APM_TRACING_LOGS_INJECTION])
+        test_agent.assert_rc_capabilities({Capabilities.APM_TRACING_LOGS_INJECTION})
 
     @irrelevant(library="cpp", reason="The CPP tracer doesn't support automatic logs injection")
     @parametrize("library_env", [{**DEFAULT_ENVVARS}])
     def test_capability_tracing_http_header_tags(self, library_env, test_agent, test_library):
         """Ensure the RC request contains the http header tags capability."""
-        test_agent.wait_for_rc_capabilities([Capabilities.APM_TRACING_HTTP_HEADER_TAGS])
+        test_agent.assert_rc_capabilities({Capabilities.APM_TRACING_HTTP_HEADER_TAGS})
 
     @parametrize("library_env", [{**DEFAULT_ENVVARS}])
     def test_capability_tracing_custom_tags(self, library_env, test_agent, test_library):
         """Ensure the RC request contains the custom tags capability."""
-        test_agent.wait_for_rc_capabilities([Capabilities.APM_TRACING_CUSTOM_TAGS])
+        test_agent.assert_rc_capabilities({Capabilities.APM_TRACING_CUSTOM_TAGS})
 
 
 @scenarios.parametric
@@ -528,7 +622,7 @@ class TestDynamicConfigSamplingRules:
     @parametrize("library_env", [{**DEFAULT_ENVVARS}])
     def test_capability_tracing_sample_rules(self, library_env, test_agent, test_library):
         """Ensure the RC request contains the trace sampling rules capability."""
-        test_agent.wait_for_rc_capabilities([Capabilities.APM_TRACING_SAMPLE_RULES])
+        test_agent.assert_rc_capabilities({Capabilities.APM_TRACING_SAMPLE_RULES})
 
     @parametrize(
         "library_env",

utils/_context/core.py

@@ -6,6 +6,8 @@
 
 import json
 
+from utils._context.library_version import LibraryVersion
+
 
 class _Context:
     """Context is an helper class that exposes scenario properties
@@ -42,7 +44,7 @@ def uds_socket(self):
         return self._get_scenario_property("uds_socket", None)
 
     @property
-    def library(self):
+    def library(self) -> LibraryVersion | None:
         return self._get_scenario_property("library", None)
 
     @property