fix stop tokens, can probably be better though

twinnydotdev · Sep 25, 2024 · 1458aa1 · 1458aa1
1 parent 6c937a6
commit 1458aa1
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/common/constants.ts b/src/common/constants.ts
@@ -220,7 +220,7 @@ export const STOP_DEEPSEEK = [
   '<｜end▁of▁sentence｜>'
 ]
 
-export const STOP_STARCODER_QWEN = ['<|endoftext|>', '<file_sep>', '<fim_prefix>']
+export const STOP_STARCODER_QWEN = ['<file_sep>', '<fim_prefix>', '<|file_sep|>', '<|fim_prefix|>', '<|endoftext|>']
 
 export const STOP_CODEGEMMA = ['<|file_separator|>', '<|end_of_turn|>', '<eos>']