Skip to content

Commit

Permalink
Add count tokens function
Browse files Browse the repository at this point in the history
I don't actually want to encode into tokens for my use case, quickly count to check my request won't exceed the limit.

This should be faster since we don't initialize the memory for the output array.

```
const crypto = require('crypto');
// Generate a random string of a given length
function generateRandomString(length) {
    return crypto.randomBytes(length).toString('hex');
}

const {encode, decode, countTokens} = require('gpt-3-encoder')

let str = 'This is an example sentence to try encoding out on!'
// let now = Date.now();
let encoded = encode(str)
console.log('Encoded this string looks like: ', encoded)
console.log('We can look at each token and what it represents)
let tokencount = 0;
for(let token of encoded){
    tokencount ++;
    console.log({token, string: decode([token])})
}
console.log("there are n tokens: ", tokencount);
let decoded = decode(encoded)
console.log('We can decode it back into:\n', decoded)

let now = Date.now();
// todo: write an benchmark for the above method vs  int countTokens(str)
str = generateRandomString(10000);

console.time('fencode');
encoded = encode(str);
console.log(`First encode to cache string n stuff in mem`);
console.timeEnd('fencode');


console.log(`Original string length: ${str.length}`);
// Benchmark the encode function
console.time('encode');
encoded = encode(str);
console.log(`Encoded string length: ${encoded.length}`);
console.timeEnd('encode');

// Benchmark the countTokens function
console.time('countTokens');
let tokenCount = countTokens(str);
console.log(`Number of tokens: ${tokenCount}`);
console.timeEnd('countTokens');


console.log(`Original string length: ${str.length}`);
console.log(`Encoded string length: ${encoded.length}`);
console.log(`Number of tokens: ${tokenCount}`);
```

```
We can decode it back into:
 This is an example sentence to try encoding out on!
First encode to cache string n stuff in mem
fencode: 163.57ms
Original string length: 20000
Encoded string length: 11993
encode: 124.265ms
Number of tokens: 11993
countTokens: 29.2ms
Original string length: 20000
Encoded string length: 11993
Number of tokens: 11993

```
  • Loading branch information
syonfox authored Dec 20, 2022
1 parent 0adbb50 commit 473daf7
Showing 1 changed file with 20 additions and 2 deletions.
22 changes: 20 additions & 2 deletions Encoder.js
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,23 @@ function encode(text) {
return bpe_tokens
}

// This function works by iterating through the matches of the pat pattern in the input text,
// encoding each match using the encodeStr function and the byte_encoder mapping,
// and then applying the bpe function to the encoded token. The number of tokens produced by the bpe function is then added to the count variable.
// Finally, the count variable is returned as the result.
function countTokens(text) {
let count = 0
const matches = Array.from(text.matchAll(pat)).map(x => x[0])
for (let token of matches) {
token = encodeStr(token).map(x => {
return byte_encoder[x]
}).join('')

count += bpe(token).split(' ').length
}
return count
}

function decode(tokens) {
let text = tokens.map(x => decoder[x]).join('')
text = decodeStr(text.split('').map(x => byte_decoder[x]))
Expand All @@ -174,5 +191,6 @@ function decode(tokens) {

module.exports = {
encode,
decode
};
decode,
countTokens
};

0 comments on commit 473daf7

Please sign in to comment.