Merge pull request #1 from NickHeiner/master

merge back the one other change and follow nick with moving to version 1.2 for feature add.

Encoder.js

@@ -82,11 +82,11 @@ const byte_decoder = {}
 Object.keys(byte_encoder).map(x => { byte_decoder[byte_encoder[x]] = x })
 
 const bpe_ranks = dictZip(bpe_merges, range(0, bpe_merges.length))
-const cache = {}
+const cache = new Map;
 
 function bpe(token) {
-  if (token in cache) {
-    return cache[token]
+  if (cache.has(token)) {
+    return cache.get(token)
   }``
 
   let word = token.split('')
@@ -147,7 +147,7 @@ function bpe(token) {
   }
 
   word = word.join(' ')
-  cache[token] = word
+  cache.set(token, word)
 
   return word
 }

Encoder.test.js

@@ -34,4 +34,11 @@ test('emojis', () => {
   const str = "hello 👋 world 🌍";
   expect(encode(str)).toEqual([31373, 50169, 233, 995, 12520, 234, 235])
   expect(decode(encode(str))).toEqual(str)
-});
+});
+
+test('properties of Object',()=>{
+	const str = "toString constructor hasOwnProperty valueOf";
+
+	expect(encode(str)).toEqual([1462, 10100, 23772, 468, 23858, 21746, 1988, 5189]);
+	expect(decode(encode(str))).toEqual(str);
+})

README.md

@@ -1,3 +1,7 @@
+# This is a fork of https://github.com/latitudegames/GPT-3-Encoder. I made this fork so I could apply some PRs that had been sent to the upstream repo.
+
+~~~
+
 # GPT-3-Encoder
 Javascript BPE Encoder Decoder for GPT-2 / GPT-3