Update to 4k seq length + lower pages + adjust tokenizer.

constants/__init__.py

@@ -31,10 +31,10 @@
 SUBNET_UID = 9
 # The root directory of this project.
 ROOT_DIR = Path(__file__).parent.parent
-# Block at which 7b models, 8192 sequence lengths, new tokenizer, bfloat16, and flash attention are used.
+# Block at which 7b models, 4096 sequence lengths, new tokenizer, bfloat16, and flash attention are used.
 BLOCK_7B = 2_735_661
 SEQUENCE_LENGTH_1 = 1024
-SEQUENCE_LENGTH_2 = 8192
+SEQUENCE_LENGTH_2 = 4096
 # A mapping of block numbers to the supported model types as of that block.
 ALLOWED_MODEL_TYPES_1 = {
     GPT2LMHeadModel,
@@ -75,7 +75,7 @@
             max_model_bytes=15 * 1024 * 1024 * 1024,
             max_model_parameters=6_900_000_000,
             allowed_model_types=ALLOWED_MODEL_TYPES_2,
-            tokenizer_identifier=TokenizerIdentifier.GPT3_5_TURBO_16K,
+            tokenizer_identifier=TokenizerIdentifier.GPT3_5_TURBO,
         ),
     ),
 ]
@@ -97,7 +97,7 @@
 # validator score boosting for earlier models.
 timestamp_epsilon = 0.005
 # validators number of pages to eval over miners on each step.
-n_eval_pages = 24
+n_eval_pages = 12
 # validator eval batch size.
 batch_size = 1
 # validator eval batch min to keep for next loop.

model/data.py

@@ -75,7 +75,7 @@ class TokenizerIdentifier(IntEnum):
     """Identifiers the tokenizer to use. This may mean different tokenizers or different implementations."""
 
     DISTILGPT_2 = 1
-    GPT3_5_TURBO_16K = 2
+    GPT3_5_TURBO = 2
 
 
 @dataclasses.dataclass()

pretrain/model.py

@@ -49,7 +49,7 @@ def get_old_tokenizer(cache_dir: str = None):
 def get_tokenizer(cache_dir: str = None):
     """Returns the tokenizer used by the latest models."""
     tokenizer = GPT2TokenizerFast.from_pretrained(
-        "Xenova/gpt-3.5-turbo-16k", cache_dir=cache_dir
+        "Xenova/gpt-3.5-turbo", cache_dir=cache_dir
     )
     tokenizer.pad_token = tokenizer.eos_token
     return tokenizer

tests/model/test_model_utils.py

@@ -14,12 +14,12 @@ class TestModelUtils(unittest.TestCase):
         tokenizer_identifier=TokenizerIdentifier.DISTILGPT_2,
     )
     MODEL_CRITERIA_7B = ModelCriteria(
-        sequence_length=8192,
+        sequence_length=4096,
         optimized=True,
         max_model_bytes=15 * 1024 * 1024 * 1024,
         max_model_parameters=6_900_000_000,
         allowed_model_types=ALLOWED_MODEL_TYPES_2,
-        tokenizer_identifier=TokenizerIdentifier.GPT3_5_TURBO_16K,
+        tokenizer_identifier=TokenizerIdentifier.GPT3_5_TURBO,
     )
     model_criteria_cases = [
         (2_405_920, MODEL_CRITERIA_772M),