From 01cfa4efd22a2899c20f1d00c73856d94bcf3d7d Mon Sep 17 00:00:00 2001 From: Steffen Cruz Date: Sat, 17 Feb 2024 15:57:35 -0600 Subject: [PATCH 1/6] Update mock objects for more control and improved testing --- prompting/mock.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/prompting/mock.py b/prompting/mock.py index e5862c27..8e7d9ae3 100644 --- a/prompting/mock.py +++ b/prompting/mock.py @@ -74,8 +74,12 @@ def preprocess(self, **kwargs): class MockSubtensor(bt.MockSubtensor): - def __init__(self, netuid, n=16, wallet=None, network="mock"): - super().__init__(network=network) + def __init__(self, netuid, n=16, wallet=None): + + super().__init__() + # reset the underlying subtensor state + self.chain_state = None + self.setup() if not self.subnet_exists(netuid): self.create_subnet(netuid) @@ -102,6 +106,10 @@ def __init__(self, netuid, n=16, wallet=None, network="mock"): class MockMetagraph(bt.metagraph): + + default_ip = "127.0.0.0" + default_port = 8091 + def __init__(self, netuid=1, network="mock", subtensor=None): super().__init__( netuid=netuid, network=network, sync=False @@ -112,17 +120,17 @@ def __init__(self, netuid=1, network="mock", subtensor=None): self.sync(subtensor=subtensor) for axon in self.axons: - axon.ip = "127.0.0.0" - axon.port = 8091 - - bt.logging.info(f"Metagraph: {self}") - bt.logging.info(f"Axons: {self.axons}") + axon.ip = self.default_ip + axon.port = self.default_port class MockDendrite(bt.dendrite): """ Replaces a real bittensor network request with a mock request that just returns some static completion for all axons that are passed and adds some random delay. """ + min_time: float = 0 + max_time: float = 1 + def __init__(self, wallet): super().__init__(wallet) @@ -145,24 +153,24 @@ async def query_all_axons(streaming: bool): async def single_axon_response(i, axon): """Queries a single axon for a response.""" - start_time = time.time() + t0 = time.time() s = synapse.copy() # Attach some more required data so it looks real s = self.preprocess_synapse_for_request(axon, s, timeout) # We just want to mock the response, so we'll just fill in some data - process_time = random.random() + process_time = random.random()*(self.max_time-self.min_time) + self.min_time + await asyncio.sleep(process_time) if process_time < timeout: - s.dendrite.process_time = str(time.time() - start_time) # Update the status code and status message of the dendrite to match the axon s.completion = f'Mock miner completion {i}' s.dendrite.status_code = 200 s.dendrite.status_message = "OK" - synapse.dendrite.process_time = str(process_time) else: s.completion = "" s.dendrite.status_code = 408 s.dendrite.status_message = "Timeout" - synapse.dendrite.process_time = str(timeout) + + s.dendrite.process_time = str(time.time() - t0) # Return the updated synapse object after deserializing if requested if deserialize: From e61c512674feedf3bc00d3ebf433a805338172cb Mon Sep 17 00:00:00 2001 From: Steffen Cruz Date: Sat, 17 Feb 2024 15:58:02 -0600 Subject: [PATCH 2/6] Add tests for mock objects --- tests/test_mock.py | 92 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 tests/test_mock.py diff --git a/tests/test_mock.py b/tests/test_mock.py new file mode 100644 index 00000000..6b3022dd --- /dev/null +++ b/tests/test_mock.py @@ -0,0 +1,92 @@ +import pytest +import asyncio +import bittensor as bt +from prompting.mock import MockDendrite, MockMetagraph, MockSubtensor +from prompting.protocol import PromptingSynapse + +@pytest.mark.parametrize('netuid', [1, 2, 3]) +@pytest.mark.parametrize('n', [2, 4, 8, 16, 32, 64]) +@pytest.mark.parametrize('wallet', [bt.MockWallet(), None]) +def test_mock_subtensor(netuid, n, wallet): + + subtensor = MockSubtensor(netuid=netuid, n=n, wallet=wallet) + neurons = subtensor.neurons(netuid=netuid) + # Check netuid + assert subtensor.subnet_exists(netuid) + # Check network + assert subtensor.network == 'mock' + assert subtensor.chain_endpoint == 'mock_endpoint' + # Check number of neurons + assert len(neurons) == (n + 1 if wallet is not None else n) + # Check wallet + if wallet is not None: + assert subtensor.is_hotkey_registered(netuid=netuid, hotkey_ss58=wallet.hotkey.ss58_address) + + for neuron in neurons: + assert type(neuron) == bt.NeuronInfo + assert subtensor.is_hotkey_registered(netuid=netuid, hotkey_ss58=neuron.hotkey) + +@pytest.mark.parametrize('n', [16, 32, 64]) +def test_mock_metagraph(n): + mock_subtensor = MockSubtensor(netuid=1, n=n) + mock_metagraph = MockMetagraph(subtensor=mock_subtensor) + # Check axons + axons = mock_metagraph.axons + assert len(axons) == n + # Check ip and port + for axon in axons: + assert type(axon) == bt.AxonInfo + assert axon.ip == mock_metagraph.default_ip + assert axon.port == mock_metagraph.default_port + +def test_mock_reward_pipeline(): + pass + +def test_mock_neuron(): + pass + +@pytest.mark.parametrize('timeout', [0.1, 0.2]) +@pytest.mark.parametrize('min_time', [0, 0.05, 0.1]) +@pytest.mark.parametrize('max_time', [0.1, 0.15, 0.2]) +@pytest.mark.parametrize('n', [4, 16, 64]) +def test_mock_dendrite_timings(timeout, min_time, max_time, n): + + mock_wallet = None#bt.MockWallet() + mock_dendrite = MockDendrite(mock_wallet) + mock_dendrite.min_time = min_time + mock_dendrite.max_time = max_time + mock_subtensor = MockSubtensor(netuid=1, n=n) + mock_metagraph = MockMetagraph(subtensor=mock_subtensor) + axons = mock_metagraph.axons + + async def run(): + return await mock_dendrite( + axons, + synapse = PromptingSynapse(roles=["user"], messages=["What is the capital of France?"]), + timeout = timeout + ) + + responses = asyncio.run(run()) + for synapse in responses: + assert hasattr(synapse, 'dendrite') and type(synapse.dendrite) == bt.TerminalInfo + + dendrite = synapse.dendrite + # check synapse.dendrite has (process_time, status_code, status_message) + for field in ('process_time', 'status_code', 'status_message'): + assert hasattr(dendrite, field) and getattr(dendrite, field) is not None + + # check that the dendrite take between min_time and max_time + assert min_time <= dendrite.process_time + assert dendrite.process_time <= max_time + 0.1 + # check that responses which take longer than timeout have 408 status code + if dendrite.process_time >= timeout + 0.1: + assert dendrite.status_code == 408 + assert dendrite.status_message == 'Timeout' + assert synapse.completion == '' + # check that responses which take less than timeout have 200 status code + elif dendrite.process_time < timeout: + assert dendrite.status_code == 200 + assert dendrite.status_message == 'OK' + # check that completions are not empty for successful responses + assert type(synapse.completion) == str and len(synapse.completion) > 0 + # dont check for responses which take between timeout and max_time because they are not guaranteed to have a status code of 200 or 408 From cfae3bb38458a514a1ddaff369ebea4d9e9cb0b6 Mon Sep 17 00:00:00 2001 From: Steffen Cruz Date: Sat, 17 Feb 2024 16:03:23 -0600 Subject: [PATCH 3/6] Remove comment --- tests/test_mock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_mock.py b/tests/test_mock.py index 6b3022dd..7cb52bd2 100644 --- a/tests/test_mock.py +++ b/tests/test_mock.py @@ -51,7 +51,7 @@ def test_mock_neuron(): @pytest.mark.parametrize('n', [4, 16, 64]) def test_mock_dendrite_timings(timeout, min_time, max_time, n): - mock_wallet = None#bt.MockWallet() + mock_wallet = None mock_dendrite = MockDendrite(mock_wallet) mock_dendrite.min_time = min_time mock_dendrite.max_time = max_time From 811d93ab8cd57a0482052175eaa449b61a741977 Mon Sep 17 00:00:00 2001 From: Steffen Cruz Date: Mon, 19 Feb 2024 13:39:14 -0600 Subject: [PATCH 4/6] Ensure a default wallet exists in test environemt --- tests/test_mock.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_mock.py b/tests/test_mock.py index 7cb52bd2..0f6c2042 100644 --- a/tests/test_mock.py +++ b/tests/test_mock.py @@ -4,9 +4,12 @@ from prompting.mock import MockDendrite, MockMetagraph, MockSubtensor from prompting.protocol import PromptingSynapse +wallet = bt.MockWallet() +wallet.create(coldkey_use_password=False) + @pytest.mark.parametrize('netuid', [1, 2, 3]) @pytest.mark.parametrize('n', [2, 4, 8, 16, 32, 64]) -@pytest.mark.parametrize('wallet', [bt.MockWallet(), None]) +@pytest.mark.parametrize('wallet', [wallet, None]) def test_mock_subtensor(netuid, n, wallet): subtensor = MockSubtensor(netuid=netuid, n=n, wallet=wallet) From 565212cec07be31a9ddbd4467b04f11b590fc0bc Mon Sep 17 00:00:00 2001 From: Steffen Cruz Date: Mon, 19 Feb 2024 13:40:07 -0600 Subject: [PATCH 5/6] Add pre-staging to workflows --- .github/workflows/python-package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 5db44d16..cedb6b55 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -5,9 +5,9 @@ name: Python package on: push: - branches: [ "main", "staging" ] + branches: [ "main", "staging", "pre-staging" ] pull_request: - branches: [ "main", "staging" ] + branches: [ "main", "staging", "pre-staging" ] jobs: build: From dd74a4ccb74076f6c8a8ad00a43f1b00b9540567 Mon Sep 17 00:00:00 2001 From: p-ferreira Date: Tue, 20 Feb 2024 19:28:04 +0000 Subject: [PATCH 6/6] fix broken unit test by adding a mock wallet to mock dendrite --- tests/test_mock.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_mock.py b/tests/test_mock.py index 0f6c2042..5bcba1ab 100644 --- a/tests/test_mock.py +++ b/tests/test_mock.py @@ -53,8 +53,7 @@ def test_mock_neuron(): @pytest.mark.parametrize('max_time', [0.1, 0.15, 0.2]) @pytest.mark.parametrize('n', [4, 16, 64]) def test_mock_dendrite_timings(timeout, min_time, max_time, n): - - mock_wallet = None + mock_wallet = bt.MockWallet(config=None) mock_dendrite = MockDendrite(mock_wallet) mock_dendrite.min_time = min_time mock_dendrite.max_time = max_time