Add text embedding models

zurawiki · Mar 20, 2024 · eb9c1de · eb9c1de
1 parent 14821ea
commit eb9c1de
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/tiktoken-rs/src/tokenizer.rs b/tiktoken-rs/src/tokenizer.rs
@@ -68,6 +68,8 @@ const MODEL_TO_TOKENIZER: &[(&str, Tokenizer)] = &[
     ("code-davinci-edit-001", Tokenizer::P50kEdit),
     // embeddings
     ("text-embedding-ada-002", Tokenizer::Cl100kBase),
+    ("text-embedding-3-small", Tokenizer::Cl100kBase),
+    ("text-embedding-3-large", Tokenizer::Cl100kBase),
     // old embeddings
     ("text-similarity-davinci-001", Tokenizer::R50kBase),
     ("text-similarity-curie-001", Tokenizer::R50kBase),