tokenizer setings

elizaOS · Dec 31, 2024 · af97657 · af97657
1 parent fc638e8
commit af97657
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/.env.example b/.env.example
@@ -356,3 +356,7 @@ CRONOSZKEVM_PRIVATE_KEY=
 
 # Fuel Ecosystem (FuelVM)
 FUEL_WALLET_PRIVATE_KEY=
+
+# Tokenizer Settings
+TOKENIZER_MODEL=  # Specify the tokenizer model to be used.
+TOKENIZER_TYPE=   # Options: tiktoken (for OpenAI models) or auto (AutoTokenizer from Hugging Face for non-OpenAI models). Default: tiktoken.