adds timeout on forward pass

prompting/base/validator.py

@@ -135,17 +135,20 @@ def run(self):
         try:
             while True:
                 bt.logging.info(f"step({self.step}) block({self.block})")
-
-                # Run multiple forwards concurrently.
+                
+                forward_timeout = self.config.neuron.forward_max_time                
                 try:
                     task = self.loop.create_task(self.forward())
-                    self.loop.run_until_complete(task)
+                    self.loop.run_until_complete(asyncio.wait_for(task, timeout=forward_timeout))
                 except torch.cuda.OutOfMemoryError as e:
                     bt.logging.error(f"Out of memory error: {e}")
                     continue
                 except MaxRetryError as e:
                     bt.logging.error(f"MaxRetryError: {e}")
                     continue
+                except asyncio.TimeoutError as e:
+                    bt.logging.error(f"Forward timeout: Task execution exceeded {forward_timeout} seconds and was cancelled.: {e}")
+                    continue
 
                 # Check if we should exit.
                 if self.should_exit:

prompting/utils/config.py

@@ -381,6 +381,13 @@ def add_validator_args(cls, parser):
         help="Only query a single hotkey per ip.",
         default=False,
     )
+
+    parser.add_argument(
+        "--neuron.forward_max_time",
+        type=int,
+        help="Max time to wait for a forward call to complete in seconds.",
+        default=120,
+    )
 
 
 def config(cls):