Pre staging

.github/workflows/python-package.yml

@@ -5,9 +5,9 @@ name: Python package
 
 on:
   push:
-    branches: [ "main", "staging" ]
+    branches: [ "main", "staging", "pre-staging" ]
   pull_request:
-    branches: [ "main", "staging" ]
+    branches: [ "main", "staging", "pre-staging" ]
 
 jobs:
   build:

README.md

@@ -39,7 +39,7 @@ The design of the network's incentive mechanism is based on two important requir
 
 It is imperative that the validation process engages with miners in the same way as real users. The reasons for this are as follows:
 - Miners will compete and continuously improve at performing the validation task(s), and so this fine tuning should be aligned with the goals of the subnet.
-- It should not be possible to distinguish between validation and API client queries so that miners always serve requests (even when they do not recieve emissions for doing so).
+- It should not be possible to distinguish between validation and API client queries so that miners always serve requests (even when they do not receive emissions for doing so).
 
 In the context of this subnet, miners are required to be intelligent AI assistants that provide helpful and correct responses to a range of queries. 
 
@@ -104,7 +104,7 @@ These validators are designed to run and update themselves automatically. To run
    pm2 start run.sh --name s1_validator_autoupdate -- --wallet.name <your-wallet-name> --wallet.hotkey <your-wallet-hot-key>
    ```
 
-This will run **two** PM2 process: one for the validator which is called `s1_validator_main_process` by default (you can change this in `run.sh`), and one for the run.sh script (in step 4, we named it `s1_validator_autoupdate`). The script will check for updates every 30 minutes, if there is an update then it will pull it, install it, restart `s1_validator_main_process` and then restart itself.
+This will run **two** PM2 processes: one for the validator which is called `s1_validator_main_process` by default (you can change this in `run.sh`), and one for the run.sh script (in step 4, we named it `s1_validator_autoupdate`). The script will check for updates every 30 minutes, if there is an update then it will pull it, install it, restart `s1_validator_main_process` and then restart itself.
 
 
 

neurons/miner.py

@@ -176,11 +176,11 @@ def log_event(self, timing: float, prompt: str, completion: str, system_prompt:
 
 # This is the main function, which runs the miner.
 if __name__ == "__main__":
-    with Miner() as miner:
+    with Miner() as m:
         while True:
-            bt.logging.info("Miner running...", time.time())
+            bt.logging.info(f"Miner running:: network: {m.subtensor.network} | block: {m.block} | step: {m.step} | uid: {m.uid} | last updated: {m.block-m.metagraph.last_update[m.uid]} | trust: {m.metagraph.trust[m.uid]:.3f} | emission {m.metagraph.emission[m.uid]:.3f}")
             time.sleep(5)
 
-            if miner.should_exit:
+            if m.should_exit:
                 bt.logging.warning("Ending miner...")
                 break

neurons/miners/openai/miner.py

@@ -53,16 +53,16 @@ def __init__(self, config=None):
 
         if self.config.wandb.on:
             self.identity_tags =  ("openai_miner", ) + (self.config.neuron.model_id, )
-        
-        _ = load_dotenv(find_dotenv()) 
-        api_key = os.environ.get("OPENAI_API_KEY")        
+
+        _ = load_dotenv(find_dotenv())
+        api_key = os.environ.get("OPENAI_API_KEY")
 
         # Set openai key and other args
         self.model = ChatOpenAI(
             api_key=api_key,
             model_name=self.config.neuron.model_id,
             max_tokens = self.config.neuron.max_tokens,
-            temperature = self.config.neuron.temperature,            
+            temperature = self.config.neuron.temperature,
         )
 
         self.system_prompt = "You are a friendly chatbot who always responds concisely and helpfully. You are honest about things you don't know."
@@ -122,7 +122,7 @@ async def forward(
 
                 role = synapse.roles[-1]
                 message = synapse.messages[-1]
-                
+
                 bt.logging.debug(f"💬 Querying openai: {prompt}")
                 response = chain.invoke(
                     {"role": role, "input": message}
@@ -133,14 +133,16 @@ async def forward(
 
                 if self.config.wandb.on:
                     self.log_event(
-                        timing=synapse_latency, 
+                        timing=synapse_latency,
                         prompt=message,
                         completion=response,
                         system_prompt=self.system_prompt,
                         extra_info=self.get_cost_logging(cb)
                     )
 
             bt.logging.debug(f"✅ Served Response: {response}")
+            self.step += 1
+
             return synapse
         except Exception as e:
             bt.logging.error(f"Error in forward: {e}")
@@ -160,4 +162,4 @@ async def forward(
 
             if miner.should_exit:
                 bt.logging.warning("Ending miner...")
-                break   
+                break

neurons/miners/test/echo.py

@@ -41,7 +41,7 @@ async def forward(
     ) -> PromptingSynapse:
 
         synapse.completion = synapse.messages[-1]
-
+        self.step += 1
         return synapse
 
     async def blacklist(

neurons/miners/test/mock.py

@@ -41,7 +41,7 @@ async def forward(
     ) -> PromptingSynapse:
 
         synapse.completion = f'Hey you reached mock miner {self.config.wallet.hotkey!r}. Please leave a message after the tone.. Beep!'
-
+        self.step += 1
         return synapse
 
     async def blacklist(

neurons/miners/test/phrase.py

@@ -54,7 +54,7 @@ async def forward(
     ) -> PromptingSynapse:
 
         synapse.completion = self.config.neuron.phrase
-
+        self.step += 1
         return synapse
 
     async def blacklist(

neurons/miners/wiki_agent/miner.py

@@ -29,7 +29,7 @@
 
 class WikipediaAgentMiner(Miner):
     """Langchain-based miner which uses OpenAI's API as the LLM. This uses the ReAct framework.
-    
+
     You should also install the dependencies for this miner, which can be found in the requirements.txt file in this directory.
     """
     @classmethod
@@ -41,14 +41,14 @@ def add_args(cls, parser: argparse.ArgumentParser):
 
     def __init__(self, config=None):
         super().__init__(config=config)
-        
+
         bt.logging.info(f"🤖📖 Initializing wikipedia agent with model {self.config.neuron.model_id}...")
 
         if self.config.wandb.on:
             self.identity_tags = ("wikipedia_agent_miner", ) + (self.config.neuron.model_id, )
-        
-        _ = load_dotenv(find_dotenv()) 
-                
+
+        _ = load_dotenv(find_dotenv())
+
         self.agent = WikiAgent(self.config.neuron.model_id, self.config.neuron.temperature)
         self.accumulated_total_tokens = 0
         self.accumulated_prompt_tokens = 0
@@ -99,26 +99,28 @@ async def forward(
             with get_openai_callback() as cb:
                 t0 = time.time()
                 bt.logging.debug(f"📧 Message received, forwarding synapse: {synapse}")
-                            
+
                 message = synapse.messages[-1]
-                
+
                 bt.logging.debug(f"💬 Querying openai and wikipedia: {message}")
-                
+
                 response = self.agent.run(message)
 
                 synapse.completion = response
                 synapse_latency = time.time() - t0
 
                 if self.config.wandb.on:
                     self.log_event(
-                        timing=synapse_latency, 
+                        timing=synapse_latency,
                         prompt=message,
                         completion=response,
                         system_prompt='',
                         extra_info=self.get_cost_logging(cb)
                     )
 
             bt.logging.debug(f"✅ Served Response: {response}")
+            self.step += 1
+
             return synapse
         except Exception as e:
             bt.logging.error(f"Error in forward: {e}")

neurons/miners/zephyr/miner.py

@@ -122,6 +122,7 @@ async def forward(self, synapse: PromptingSynapse) -> PromptingSynapse:
 
             bt.logging.debug(f"✅ Served Response: {response}")
             torch.cuda.empty_cache()
+            self.step += 1
 
         except Exception as e:
             bt.logging.error(f"Error: {e}")

neurons/validator.py

@@ -55,7 +55,7 @@ def __init__(self, config=None):
             if p > 0
         ]
         # Load the reward pipeline
-        self.reward_pipeline = RewardPipeline(selected_tasks=self.active_tasks, device=self.device)        
+        self.reward_pipeline = RewardPipeline(selected_tasks=self.active_tasks, device=self.device)
 
     async def forward(self):
         """
@@ -100,12 +100,12 @@ def __exit__(self, exc_type, exc_value, traceback):
 
 # The main function parses the configuration and runs the validator.
 if __name__ == "__main__":
-    with Validator() as validator:
+    with Validator() as v:
         while True:
-            bt.logging.info("Validator running...", time.time())
+            bt.logging.info(f"Validator running:: network: {v.subtensor.network} | block: {v.block} | step: {v.step} | uid: {v.uid} | last updated: {v.block-v.metagraph.last_update[v.uid]} | vtrust: {v.metagraph.validator_trust[v.uid]:.3f} | emission {v.metagraph.emission[v.uid]:.3f}")
             time.sleep(5)
 
-            if validator.should_exit:
+            if v.should_exit:
                 bt.logging.warning("Ending validator...")
                 break
-            
+

prompting/__init__.py

@@ -16,7 +16,7 @@
 # DEALINGS IN THE SOFTWARE.
 
 # Define the version of the template module.
-__version__ = "1.0.4"
+__version__ = "1.1.0"
 version_split = __version__.split(".")
 __spec_version__ = (
     (10000 * int(version_split[0]))

prompting/base/validator.py

@@ -29,18 +29,18 @@
 from prompting.base.neuron import BaseNeuron
 from prompting.mock import MockDendrite
 from prompting.utils.config import add_validator_args
-
+from prompting.utils.exceptions import MaxRetryError
 
 class BaseValidatorNeuron(BaseNeuron):
     """
     Base class for Bittensor validators. Your validator should inherit from this class.
     """
-    
+
     @classmethod
     def add_args(cls, parser: argparse.ArgumentParser):
         super().add_args(parser)
-        add_validator_args(cls, parser)    
- 
+        add_validator_args(cls, parser)
+
 
     def __init__(self, config=None):
         super().__init__(config=config)
@@ -140,7 +140,14 @@ def run(self):
                 bt.logging.info(f"step({self.step}) block({self.block})")
 
                 # Run multiple forwards concurrently.
-                self.loop.run_until_complete(self.concurrent_forward())
+                try:
+                    self.loop.run_until_complete(self.concurrent_forward())
+                except torch.cuda.OutOfMemoryError as e:
+                    bt.logging.error(f"Out of memory error: {e}")
+                    continue
+                except MaxRetryError as e:
+                    bt.logging.error(f"MaxRetryError: {e}")
+                    continue
 
                 # Check if we should exit.
                 if self.should_exit:
@@ -161,8 +168,8 @@ def run(self):
         except Exception as err:
             bt.logging.error("Error during validation", str(err))
             bt.logging.debug(print_exception(type(err), err, err.__traceback__))
-            self.should_exit = True        
-    
+            self.should_exit = True
+
     def run_in_background_thread(self):
         """
         Starts the validator's operations in a background thread upon entering the context.
@@ -323,6 +330,7 @@ def update_scores(self, rewards: torch.FloatTensor, uids: List[int]):
         # shape: [ metagraph.n ]
         alpha = self.config.neuron.moving_average_alpha
         self.scores = alpha * step_rewards + (1 - alpha) * self.scores
+        self.scores = (self.scores - self.config.neuron.decay_alpha).clamp(min=0)
         bt.logging.debug(f"Updated moving avg scores: {self.scores}")
 
     def save_state(self):

prompting/conversation.py

@@ -8,9 +8,9 @@
 )
 from prompting.tools import (
     WikiDataset,
-    CodingDataset,
+    HFCodingDataset,
     MathDataset,
-    DateQADataset,
+    WikiDateDataset,
 )
 
 from transformers import Pipeline
@@ -26,13 +26,13 @@ def create_task(llm_pipeline: Pipeline, task_name: str) -> Task:
         dataset = WikiDataset()
 
     elif task_name in coding_based_tasks:
-        dataset = CodingDataset()
+        dataset = HFCodingDataset()
 
     elif task_name == "math":
         dataset = MathDataset()
 
     elif task_name == "date_qa":
-        dataset = DateQADataset()
+        dataset = WikiDateDataset()
 
     if task_name == "summarization":
         task = SummarizationTask(llm_pipeline=llm_pipeline, context=dataset.next())

prompting/mock.py

@@ -74,8 +74,12 @@ def preprocess(self, **kwargs):
 
 
 class MockSubtensor(bt.MockSubtensor):
-    def __init__(self, netuid, n=16, wallet=None, network="mock"):
-        super().__init__(network=network)
+    def __init__(self, netuid, n=16, wallet=None):
+
+        super().__init__()
+        # reset the underlying subtensor state
+        self.chain_state = None
+        self.setup()
 
         if not self.subnet_exists(netuid):
             self.create_subnet(netuid)
@@ -102,6 +106,10 @@ def __init__(self, netuid, n=16, wallet=None, network="mock"):
 
 
 class MockMetagraph(bt.metagraph):
+
+    default_ip = "127.0.0.0"
+    default_port = 8091
+
     def __init__(self, netuid=1, network="mock", subtensor=None):
         super().__init__(
             netuid=netuid, network=network, sync=False
@@ -112,17 +120,17 @@ def __init__(self, netuid=1, network="mock", subtensor=None):
         self.sync(subtensor=subtensor)
 
         for axon in self.axons:
-            axon.ip = "127.0.0.0"
-            axon.port = 8091
-
-        bt.logging.info(f"Metagraph: {self}")
-        bt.logging.info(f"Axons: {self.axons}")
+            axon.ip = self.default_ip
+            axon.port = self.default_port
 
 
 class MockDendrite(bt.dendrite):
     """
     Replaces a real bittensor network request with a mock request that just returns some static completion for all axons that are passed and adds some random delay.
     """
+    min_time: float = 0
+    max_time: float = 1
+
     def __init__(self, wallet):
         super().__init__(wallet)
 
@@ -145,24 +153,24 @@ async def query_all_axons(streaming: bool):
             async def single_axon_response(i, axon):
                 """Queries a single axon for a response."""
 
-                start_time = time.time()
+                t0 = time.time()
                 s = synapse.copy()
                 # Attach some more required data so it looks real
                 s = self.preprocess_synapse_for_request(axon, s, timeout)
                 # We just want to mock the response, so we'll just fill in some data
-                process_time = random.random()
+                process_time = random.random()*(self.max_time-self.min_time) + self.min_time
+                await asyncio.sleep(process_time)
                 if process_time < timeout:
-                    s.dendrite.process_time = str(time.time() - start_time)
                     # Update the status code and status message of the dendrite to match the axon
                     s.completion = f'Mock miner completion {i}'
                     s.dendrite.status_code = 200
                     s.dendrite.status_message = "OK"
-                    synapse.dendrite.process_time = str(process_time)
                 else:
                     s.completion = ""
                     s.dendrite.status_code = 408
                     s.dendrite.status_message = "Timeout"
-                    synapse.dendrite.process_time = str(timeout)
+
+                s.dendrite.process_time = str(time.time() - t0)
 
                 # Return the updated synapse object after deserializing if requested
                 if deserialize: