Merge pull request latitudegames#12 from andrew-healey/master

Fix bug in BPE cache
syonfox · Jan 13, 2023 · 8d31919 · 8d31919
2 parents 0adbb50 + e81024a
commit 8d31919
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 5 deletions.
diff --git a/Encoder.js b/Encoder.js
@@ -82,11 +82,11 @@ const byte_decoder = {}
 Object.keys(byte_encoder).map(x => { byte_decoder[byte_encoder[x]] = x })
 
 const bpe_ranks = dictZip(bpe_merges, range(0, bpe_merges.length))
-const cache = {}
+const cache = new Map;
 
 function bpe(token) {
-  if (token in cache) {
-    return cache[token]
+  if (cache.has(token)) {
+    return cache.get(token)
   }``
 
   let word = token.split('')
@@ -147,7 +147,7 @@ function bpe(token) {
   }
 
   word = word.join(' ')
-  cache[token] = word
+  cache.set(token, word)
 
   return word
 }

diff --git a/Encoder.test.js b/Encoder.test.js
@@ -34,4 +34,11 @@ test('emojis', () => {
   const str = "hello 👋 world 🌍";
   expect(encode(str)).toEqual([31373, 50169, 233, 995, 12520, 234, 235])
   expect(decode(encode(str))).toEqual(str)
-});
+});
+
+test('properties of Object',()=>{
+	const str = "toString constructor hasOwnProperty valueOf";
+
+	expect(encode(str)).toEqual([1462, 10100, 23772, 468, 23858, 21746, 1988, 5189]);
+	expect(decode(encode(str))).toEqual(str);
+})