v2.17.6

.gitignore

@@ -183,3 +183,4 @@ wandb
 .vscode
 **/api_keys.json
 weights.csv
+past_websites.csv

neurons/miners/epistula_miner/miner.py

@@ -45,7 +45,11 @@ def __init__(self):
             },
         )
         if SHOULD_SERVE_LLM:
-            self.llm = ReproducibleHF(model_id=LOCAL_MODEL_ID)
+            self.llm = ReproducibleHF(
+                model_id=LOCAL_MODEL_ID,
+                device=shared_settings.NEURON_DEVICE,
+                sampling_params=shared_settings.SAMPLING_PARAMS,
+            )
         else:
             self.llm = None
 

neurons/validator.py

@@ -72,7 +72,17 @@ async def spawn_loops(task_queue, scoring_queue, reward_events):
             logger.debug(f"Number of tasks in Scoring Queue: {len(scoring_queue)}")
             logger.debug(f"Number of tasks in Reward Events: {len(reward_events)}")
 
-    asyncio.run(spawn_loops(task_queue, scoring_queue, reward_events))
+    try:
+        asyncio.run(spawn_loops(task_queue, scoring_queue, reward_events))
+    except Exception as e:
+        logger.info(f"Terminating loop process: {e}")
+    finally:
+        logger.info("Cleaning up resources...")
+
+        # Ensure wandb is closed properly
+        if settings.shared_settings.WANDB_ON:
+            wandb.finish()
+            logger.info("WandB run finished.")
 
 
 def start_api(scoring_queue, reward_events):
@@ -150,19 +160,21 @@ async def main():
                         f"Metagraph hasn't been updated for {current_block - last_update_block} blocks. "
                         f"Staled block: {current_block}, Last update: {last_update_block}"
                     )
-                    sys.exit(1)
+                    break  # Exit the loop
                 step += 1
 
+        except KeyboardInterrupt:
+            logger.info("KeyboardInterrupt detected. Shutting down gracefully...")
         except Exception as e:
             logger.error(f"Main loop error: {e}")
             raise
         finally:
-            wandb.teardown()
             # Clean up processes
             for process in processes:
                 if process.is_alive():
                     process.terminate()
                     process.join()
+            sys.exit(1)
 
 
 # The main function parses the configuration and runs the validator.

prompting/llms/hf_llm.py

@@ -4,38 +4,38 @@
 import torch
 from transformers import AutoModelForCausalLM, AutoTokenizer, PreTrainedModel, pipeline
 
-from shared.settings import shared_settings
-
 
 class ReproducibleHF:
-    def __init__(self, model_id="hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4", **kwargs):
-        """
-        Initialize Hugging Face model with reproducible settings and optimizations
-        """
-        # Create a random seed for reproducibility
-        # self.seed = random.randint(0, 1_000_000)
-        # self.set_random_seeds(self.seed)
+    def __init__(
+        self,
+        model_id: str = "hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4",
+        device: str = "cuda:0",
+        sampling_params: dict[str, str | float | int | bool] | None = None,
+    ):
+        """Deterministic HuggingFace model."""
+        self._device = device
+        self.sampling_params = {} if sampling_params is None else sampling_params
         self.model: PreTrainedModel = AutoModelForCausalLM.from_pretrained(
             model_id,
             torch_dtype=torch.float16,
             low_cpu_mem_usage=True,
-            device_map="cuda:0",
+            device_map=self._device,
         )
 
         self.tokenizer = AutoTokenizer.from_pretrained(model_id)
         self.valid_generation_params = set(
             AutoModelForCausalLM.from_pretrained(model_id).generation_config.to_dict().keys()
         )
-
         self.llm = pipeline("text-generation", model=self.model, tokenizer=self.tokenizer)
 
-        self.sampling_params = shared_settings.SAMPLING_PARAMS
-
     @torch.inference_mode()
-    def generate(self, messages: list[str] | list[dict], sampling_params=None, seed=None):
-        """
-        Generate text with optimized performance
-        """
+    def generate(
+        self,
+        messages: list[str] | list[dict[str, str]],
+        sampling_params: dict[str, str | float | int | bool] | None = None,
+        seed: int | None = None,
+    ) -> str:
+        """Generate text with optimized performance."""
         self.set_random_seeds(seed)
 
         inputs = self.tokenizer.apply_chat_template(
@@ -44,14 +44,13 @@ def generate(self, messages: list[str] | list[dict], sampling_params=None, seed=
             add_generation_prompt=True,
             return_tensors="pt",
             return_dict=True,
-        ).to(shared_settings.NEURON_DEVICE)
+        ).to(self._device)
 
         params = sampling_params if sampling_params else self.sampling_params
         filtered_params = {k: v for k, v in params.items() if k in self.valid_generation_params}
 
-        # Generate with optimized settings
         outputs = self.model.generate(
-            **inputs.to(shared_settings.NEURON_DEVICE),
+            **inputs,
             **filtered_params,
             eos_token_id=self.tokenizer.eos_token_id,
         )
@@ -61,21 +60,10 @@ def generate(self, messages: list[str] | list[dict], sampling_params=None, seed=
             skip_special_tokens=True,
         )[0]
 
-        # logger.debug(
-        #     f"""{self.__class__.__name__} queried:
-        #     prompt: {messages}\n
-        #     responses: {results}\n
-        #     sampling params: {params}\n
-        #     seed: {seed}
-        #     """
-        # )
-
         return results if len(results) > 1 else results[0]
 
-    def set_random_seeds(self, seed=42):
-        """
-        Set random seeds for reproducibility across all relevant libraries
-        """
+    def set_random_seeds(self, seed: int | None = 42):
+        """Set random seeds for reproducibility across all relevant libraries."""
         if seed is not None:
             random.seed(seed)
             np.random.seed(seed)

prompting/llms/model_manager.py

@@ -65,9 +65,9 @@ def load_model(self, model_config: ModelConfig, force: bool = True):
             GPUInfo.log_gpu_info()
 
             model = ReproducibleHF(
-                model=model_config.llm_model_id,
-                gpu_memory_utilization=model_config.min_ram / GPUInfo.free_memory,
-                max_model_len=settings.shared_settings.LLM_MAX_MODEL_LEN,
+                model_id=model_config.llm_model_id,
+                device=settings.shared_settings.NEURON_DEVICE,
+                sampling_params=settings.shared_settings.SAMPLING_PARAMS,
             )
 
             self.active_models[model_config] = model

prompting/rewards/date.py

@@ -89,7 +89,7 @@ def date_score(self, reference: str, completion: str) -> float:
             score = 0
         return score
 
-    def reward(self, reference: str, response_event: DendriteResponseEvent, **kwargs) -> BatchRewardOutput:
+    async def reward(self, reference: str, response_event: DendriteResponseEvent, **kwargs) -> BatchRewardOutput:
         """Compute difference scores given a completion and reference pair.
 
         Args:

prompting/rewards/exact_match.py

@@ -28,7 +28,7 @@ def normalize_timing(timing: float, timings: float) -> float:
 
 
 class ExactMatchRewardModel(BaseRewardModel):
-    def reward(self, reference: str, response_event: DendriteResponseEvent, **kwargs) -> BatchRewardOutput:
+    async def reward(self, reference: str, response_event: DendriteResponseEvent, **kwargs) -> BatchRewardOutput:
         """
         Calculates rewards based on an exact match of the response with the reference string.
 

prompting/rewards/float_diff.py

@@ -55,7 +55,7 @@ def math_score(reference: str, completion: str) -> float:
         except Exception:
             return 0.0
 
-    def reward(self, reference: str, response_event: DendriteResponseEvent, **kwargs) -> BatchRewardOutput:
+    async def reward(self, reference: str, response_event: DendriteResponseEvent, **kwargs) -> BatchRewardOutput:
         """Compute difference scores given a completion and reference pair."""
         rewards = []
         timings = []

prompting/rewards/inference_reward_model.py

@@ -5,7 +5,7 @@
 
 
 class InferenceRewardModel(BaseRewardModel):
-    def reward(
+    async def reward(
         self,
         reference: str,
         response_event: DendriteResponseEvent,
@@ -14,5 +14,5 @@ def reward(
     ) -> BatchRewardOutput:
         """Gives an exact reward of 1 if the response matches the reference, 0 otherwise"""
         if model_id:
-            return ExactMatchRewardModel().reward(reference, response_event)
-        return RelevanceRewardModel().reward(reference, response_event)
+            return await ExactMatchRewardModel().reward(reference, response_event)
+        return await RelevanceRewardModel().reward(reference, response_event)

prompting/rewards/multi_choice.py

@@ -29,8 +29,8 @@ def safe_load_json(json_string: str) -> dict[str, float]:
         cleaned_json_string = re.sub(r'"\s*\n\s*"', r'""', cleaned_json_string)
         try:
             return {k.upper(): v for k, v in json.loads(cleaned_json_string).items()}
-        except json.JSONDecodeError as e:
-            raise ValueError(f"Invalid JSON data: {e}")
+        except Exception:
+            return None
 
     def process_predictions(self, predictions: dict[str, float]) -> dict[str, float]:
         if not all(isinstance(value, (int, float)) for value in predictions.values()):
@@ -56,12 +56,14 @@ def letter_reward(self, reference: str, completion: str) -> float:
     def logit_reward(self, reference: str, completion: str) -> float:
         try:
             loaded_json = self.safe_load_json(completion)
+            if not loaded_json:
+                return None
             valid_choices = self.process_predictions(loaded_json)
             return valid_choices.get(reference.upper(), 0.0)
         except ValueError:
             return None
 
-    def reward(self, reference: str, response_event: DendriteResponseEvent, **kwargs) -> BatchRewardOutput:
+    async def reward(self, reference: str, response_event: DendriteResponseEvent, **kwargs) -> BatchRewardOutput:
         rewards = []
         timings = []
 

prompting/rewards/penalty.py

@@ -13,7 +13,7 @@ class PenaltyModel(BaseRewardModel):
     def name(self) -> str:
         return "penalty"
 
-    def reward(self, reference: str, response_event: DendriteResponseEvent, **kwargs) -> BatchRewardOutput:
+    async def reward(self, reference: str, response_event: DendriteResponseEvent, **kwargs) -> BatchRewardOutput:
         """Penalises miner if they do not respond."""
         rewards = []
         timings = []

prompting/rewards/relevance.py

@@ -28,7 +28,7 @@ def init_model(self) -> "RelevanceRewardModel":
         self.embedding_model = MODEL
         return self
 
-    def reward(self, reference: str, response_event: DendriteResponseEvent, **kwargs) -> BatchRewardOutput:
+    async def reward(self, reference: str, response_event: DendriteResponseEvent, **kwargs) -> BatchRewardOutput:
         """Calculate the cosine similarity between sentence embeddings of the reference and completions.
 
         We subtract a baseline score which is what an empty string would get (a failed completion).

prompting/rewards/reward.py

@@ -69,10 +69,10 @@ class BaseRewardModel(ABC, BaseModel):
     weight: float = 1.0
 
     @abstractmethod
-    def reward(self, reference: str, response_event: DendriteResponseEvent, **kwargs) -> BatchRewardOutput:
+    async def reward(self, reference: str, response_event: DendriteResponseEvent, **kwargs) -> BatchRewardOutput:
         raise NotImplementedError("You must implement the reward method")
 
-    def apply(
+    async def apply(
         self,
         response_event: DendriteResponseEvent,
         reference: str | None = None,
@@ -83,7 +83,7 @@ def apply(
     ) -> WeightedRewardEvent:
         t0 = time.time()
         comparator = reference if reward_type == "reward" else challenge
-        batch_rewards_output: BatchRewardOutput = self.reward(comparator, response_event, task=task, **kwargs)
+        batch_rewards_output: BatchRewardOutput = await self.reward(comparator, response_event, task=task, **kwargs)
         batch_rewards_time = time.time() - t0
 
         return WeightedRewardEvent(
@@ -136,7 +136,7 @@ def final_rewards(cls, reward_events: list[WeightedRewardEvent]) -> list[float]:
         return cls.sum_rewards(reward_events)
 
     @classmethod
-    def apply(
+    async def apply(
         cls,
         response_event: DendriteResponseEvent,
         reference: str,
@@ -147,7 +147,7 @@ def apply(
         reward_events = []
         for weighted_reward in cls.reward_definitions:
             reward_events.append(
-                weighted_reward.apply(
+                await weighted_reward.apply(
                     reference=reference,
                     response_event=response_event,
                     challenge=challenge,

prompting/rewards/rouge.py

@@ -22,7 +22,7 @@ def rouge_score(self, reference, completion):
             return 0.0
         return self.rouge.get_scores(reference, completion, avg=self.avg)[0][self.ngram][self.metric]
 
-    def reward(self, reference: str, response_event: DendriteResponseEvent, **kwargs) -> BatchRewardOutput:
+    async def reward(self, reference: str, response_event: DendriteResponseEvent, **kwargs) -> BatchRewardOutput:
         """Compute ROUGE scores given a completion and reference pair."""
         rewards = []
         timings = []

prompting/rewards/scoring.py

@@ -21,7 +21,7 @@ class TaskScorer(AsyncLoopRunner):
 
     is_running: bool = False
     thread: threading.Thread = None
-    interval: int = 10
+    interval: int = 0
     scoring_queue: list | None = None
     reward_events: list | None = None
 
@@ -76,7 +76,7 @@ async def run_step(self) -> RewardLoggingEvent:
 
         # and there we then calculate the reward
         reward_pipeline = TaskRegistry.get_task_reward(scoring_config.task)
-        reward_events = reward_pipeline.apply(
+        reward_events = await reward_pipeline.apply(
             response_event=scoring_config.response,
             challenge=scoring_config.task.query,
             reference=scoring_config.task.reference,

prompting/rewards/streaming.py

@@ -17,8 +17,7 @@ def __init__(self, max_tokens_per_chunk: int, **kwargs):
         super().__init__()
         self.max_tokens_per_chunk = max_tokens_per_chunk
 
-    def reward(self, _: str, response_event: DendriteResponseEvent) -> BatchRewardOutput:
-        """Compute difference scores given a completion and reference pair."""
+    async def reward(self, reference: str, response_event: DendriteResponseEvent) -> BatchRewardOutput:
         """Compute difference scores given a completion and reference pair."""
         rewards = []
         timings = []