More robust to errors when reading logs which had caused freezes

allenai · Dec 2, 2024 · b3ca86a · b3ca86a
1 parent d4f3cff
commit b3ca86a
Showing 1 changed file with 5 additions and 2 deletions.
diff --git a/pdelfin/beakerpipeline.py b/pdelfin/beakerpipeline.py
@@ -474,8 +474,11 @@ async def read_stream(stream):
             line = await stream.readline()
             if not line:
                 break
-            line = line.decode('utf-8').rstrip()
-            await process_line(line)
+            try:
+                line = line.decode('utf-8').rstrip()
+                await process_line(line)
+            except Exception as ex:
+                logger.warning(f"Got {ex} when reading log line from inference server, skipping")
 
     async def timeout_task():
         nonlocal last_running_req, last_queue_req, last_semaphore_release