Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
I don't actually want to encode into tokens for my use case, quickly count to check my request won't exceed the limit. This should be faster since we don't initialize the memory for the output array. ``` const crypto = require('crypto'); // Generate a random string of a given length function generateRandomString(length) { return crypto.randomBytes(length).toString('hex'); } const {encode, decode, countTokens} = require('gpt-3-encoder') let str = 'This is an example sentence to try encoding out on!' // let now = Date.now(); let encoded = encode(str) console.log('Encoded this string looks like: ', encoded) console.log('We can look at each token and what it represents) let tokencount = 0; for(let token of encoded){ tokencount ++; console.log({token, string: decode([token])}) } console.log("there are n tokens: ", tokencount); let decoded = decode(encoded) console.log('We can decode it back into:\n', decoded) let now = Date.now(); // todo: write an benchmark for the above method vs int countTokens(str) str = generateRandomString(10000); console.time('fencode'); encoded = encode(str); console.log(`First encode to cache string n stuff in mem`); console.timeEnd('fencode'); console.log(`Original string length: ${str.length}`); // Benchmark the encode function console.time('encode'); encoded = encode(str); console.log(`Encoded string length: ${encoded.length}`); console.timeEnd('encode'); // Benchmark the countTokens function console.time('countTokens'); let tokenCount = countTokens(str); console.log(`Number of tokens: ${tokenCount}`); console.timeEnd('countTokens'); console.log(`Original string length: ${str.length}`); console.log(`Encoded string length: ${encoded.length}`); console.log(`Number of tokens: ${tokenCount}`); ``` ``` We can decode it back into: This is an example sentence to try encoding out on! First encode to cache string n stuff in mem fencode: 163.57ms Original string length: 20000 Encoded string length: 11993 encode: 124.265ms Number of tokens: 11993 countTokens: 29.2ms Original string length: 20000 Encoded string length: 11993 Number of tokens: 11993 ```
- Loading branch information