diff --git a/service/src/main/java/crawlercommons/urlfrontier/service/AbstractFrontierService.java b/service/src/main/java/crawlercommons/urlfrontier/service/AbstractFrontierService.java index e474688..15492b7 100644 --- a/service/src/main/java/crawlercommons/urlfrontier/service/AbstractFrontierService.java +++ b/service/src/main/java/crawlercommons/urlfrontier/service/AbstractFrontierService.java @@ -595,6 +595,12 @@ public void getURLs(GetParams request, StreamObserver responseObserver) return; } + // already has its fill of URLs in process + if (queue.getInProcess(now) >= maxURLsPerQueue) { + responseObserver.onCompleted(); + return; + } + int totalSent = sendURLsForQueue( queue, qwc, maxURLsPerQueue, secsUntilRequestable, now, synchStreamObs);