split in half if too many jars

DataDog · Oct 28, 2024 · 89764a3 · 89764a3
1 parent fdac37c
commit 89764a3
Show file tree

Hide file tree

Showing 2 changed files with 59 additions and 10 deletions.
diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/sink/SymbolSink.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/sink/SymbolSink.java
@@ -140,18 +140,34 @@ private void splitAndSend(List<Scope> scopesToSerialize) {
             continue;
           }
           LOGGER.debug("Sending {} jar scope size={}", scope.getName(), json.length());
-          updateStats(Collections.singletonList(scope), json);
-          symbolUploader.uploadAsMultipart(
-              "",
-              event,
-              new BatchUploader.MultiPartContent(
-                  json.getBytes(StandardCharsets.UTF_8), "file", "file.json"));
+          doUpload(Collections.singletonList(scope), json);
         }
       } else {
-        LOGGER.warn(
-            "Too many jar scopes to send, {} jar scopes, max is {}",
-            scopesToSerialize.size(),
-            BatchUploader.MAX_ENQUEUED_REQUESTS);
+        // split the list of jar scope in 2 list jar scopes with half of the jar scopes
+        int half = scopesToSerialize.size() / 2;
+        List<Scope> firstHalf = scopesToSerialize.subList(0, half);
+        List<Scope> secondHalf = scopesToSerialize.subList(half, scopesToSerialize.size());
+        LOGGER.debug("split jar scope list in 2: {} and {}", firstHalf.size(), secondHalf.size());
+        String jsonFirstHalf =
+            SERVICE_VERSION_ADAPTER.toJson(
+                new ServiceVersion(serviceName, env, version, "JAVA", firstHalf));
+        if (jsonFirstHalf.length() > maxPayloadSize) {
+          LOGGER.warn(
+              "Cannot split jar scope list in 2, first half is too big: {}",
+              jsonFirstHalf.length());
+          return;
+        }
+        doUpload(firstHalf, jsonFirstHalf);
+        String jsonSecondHalf =
+            SERVICE_VERSION_ADAPTER.toJson(
+                new ServiceVersion(serviceName, env, version, "JAVA", secondHalf));
+        if (jsonSecondHalf.length() > maxPayloadSize) {
+          LOGGER.warn(
+              "Cannot split jar scope list in 2, second half is too big: {}",
+              jsonSecondHalf.length());
+          return;
+        }
+        doUpload(secondHalf, jsonSecondHalf);
       }
     } else {
       Scope jarScope = scopesToSerialize.get(0);
@@ -180,6 +196,15 @@ private void splitAndSend(List<Scope> scopesToSerialize) {
     }
   }
 
+  private void doUpload(List<Scope> scopes, String json) {
+    updateStats(scopes, json);
+    symbolUploader.uploadAsMultipart(
+        "",
+        event,
+        new BatchUploader.MultiPartContent(
+            json.getBytes(StandardCharsets.UTF_8), "file", "file.json"));
+  }
+
   private static Scope createJarScope(String jarName, List<Scope> classScopes) {
     return Scope.builder(ScopeType.JAR, jarName, 0, 0).name(jarName).scopes(classScopes).build();
   }

diff --git a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/sink/SymbolSinkTest.java b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/sink/SymbolSinkTest.java
@@ -103,6 +103,30 @@ public void splitByJarScopes() {
     }
   }
 
+  @Test
+  public void splitTootManyJarScopes() {
+    SymbolUploaderMock symbolUploaderMock = new SymbolUploaderMock();
+    Config config = mock(Config.class);
+    when(config.getServiceName()).thenReturn("service1");
+    SymbolSink symbolSink = new SymbolSink(config, symbolUploaderMock, 2048);
+    final int NUM_JAR_SCOPES = 21;
+    for (int i = 0; i < NUM_JAR_SCOPES; i++) {
+      symbolSink.addScope(
+          Scope.builder(ScopeType.JAR, "jar" + i + ".jar", 0, 0)
+              .scopes(singletonList(Scope.builder(ScopeType.CLASS, "class" + i, 0, 0).build()))
+              .build());
+    }
+    symbolSink.flush();
+    // split upload request per half jar scopes
+    assertEquals(2 * 2, symbolUploaderMock.multiPartContents.size());
+    String strContent1 = assertMultipartContent(symbolUploaderMock, 0);
+    assertTrue(strContent1.contains("\"source_file\":\"jar0.jar\""));
+    assertTrue(strContent1.contains("\"source_file\":\"jar9.jar\""));
+    String strContent2 = assertMultipartContent(symbolUploaderMock, 2);
+    assertTrue(strContent2.contains("\"source_file\":\"jar10.jar\""));
+    assertTrue(strContent2.contains("\"source_file\":\"jar20.jar\""));
+  }
+
   @Test
   public void splitByClassScopes() {
     SymbolUploaderMock symbolUploaderMock = new SymbolUploaderMock();