From a730019231499e834300977b358097c8562b2243 Mon Sep 17 00:00:00 2001 From: Jasper Alexander <84851956+JasperAlexander@users.noreply.github.com> Date: Thu, 18 May 2023 20:16:06 +0200 Subject: [PATCH] Added package text-to-embeddings --- .changeset/tricky-dragons-hang.md | 5 +++ packages/text-to-embeddings/README.md | 44 ++++++++++++++++++ packages/text-to-embeddings/dist/index.d.ts | 16 +++++++ packages/text-to-embeddings/dist/index.js | 1 + packages/text-to-embeddings/package.json | 45 +++++++++++++++++++ packages/text-to-embeddings/src/index.ts | 45 +++++++++++++++++++ packages/text-to-embeddings/tsconfig.json | 13 ++++++ packages/text-to-embeddings/webpack.config.js | 24 ++++++++++ pnpm-lock.yaml | 25 +++++++++++ 9 files changed, 218 insertions(+) create mode 100644 .changeset/tricky-dragons-hang.md create mode 100644 packages/text-to-embeddings/README.md create mode 100644 packages/text-to-embeddings/dist/index.d.ts create mode 100644 packages/text-to-embeddings/dist/index.js create mode 100644 packages/text-to-embeddings/package.json create mode 100644 packages/text-to-embeddings/src/index.ts create mode 100644 packages/text-to-embeddings/tsconfig.json create mode 100644 packages/text-to-embeddings/webpack.config.js diff --git a/.changeset/tricky-dragons-hang.md b/.changeset/tricky-dragons-hang.md new file mode 100644 index 0000000..e354d35 --- /dev/null +++ b/.changeset/tricky-dragons-hang.md @@ -0,0 +1,5 @@ +--- +"text-to-embeddings": major +--- + +Added package text-to-embeddings diff --git a/packages/text-to-embeddings/README.md b/packages/text-to-embeddings/README.md new file mode 100644 index 0000000..e20f6a3 --- /dev/null +++ b/packages/text-to-embeddings/README.md @@ -0,0 +1,44 @@ +# text-to-embeddings + +A utility to convert text to vector embeddings. Currently only supports OpenAI embeddings. The utility is made to be used on a Node.js server for security reasons. + +## Installation + +Go to the root of your project directory and run the following command. + +```sh +pnpm add text-to-embeddings +``` + +## Usage + +Use text-to-embeddings in a Node.js server. In Next.js 13, you can make a route handler with the following code. + +```tsx +import { NextResponse } from 'next/server' +import { textToEmbeddings } from 'text-to-embeddings' + +export const config = { + runtime: 'edge', +} + +export async function POST(req: Request) { + const body = await req.json() + + const embeddings = await textToEmbeddings({ + apiKey: process.env.OPENAI_API_KEY, + user: body.user, + text: body.text + }) + + return new Response(embeddings) +} +``` + +## Contributing + +If you're interested in contributing, please read the [contributing docs](../../CONTRIBUTING.md) before submitting a pull request. + +## License + +This package is licensed under the MIT License. See [LICENSE](../../LICENSE.md) for more information. diff --git a/packages/text-to-embeddings/dist/index.d.ts b/packages/text-to-embeddings/dist/index.d.ts new file mode 100644 index 0000000..0c3fbd5 --- /dev/null +++ b/packages/text-to-embeddings/dist/index.d.ts @@ -0,0 +1,16 @@ +/** + * Converts text to vector embeddings, currently only supports OpenAI + * @param apiUrl The url of the embeddings API, defaults to https://api.openai.com/v1/embeddings + * @param apiKey The key of the API + * @param user A unique identifier representing the user + * @param model The model to use, defaults to text-embedding-ada-002 + * @param text The text to convert + * @returns The AI-generated text or ReadableStream + */ +export declare function textToEmbeddings({ apiUrl, apiKey, user, model, text }: { + apiUrl?: string; + apiKey?: string; + user?: string; + model?: string; + text: string; +}): Promise; diff --git a/packages/text-to-embeddings/dist/index.js b/packages/text-to-embeddings/dist/index.js new file mode 100644 index 0000000..531e711 --- /dev/null +++ b/packages/text-to-embeddings/dist/index.js @@ -0,0 +1 @@ +(()=>{"use strict";var e={};({899:function(e,t){var n=this&&this.__awaiter||function(e,t,n,i){return new(n||(n=Promise))((function(o,a){function d(e){try{u(i.next(e))}catch(e){a(e)}}function r(e){try{u(i.throw(e))}catch(e){a(e)}}function u(e){var t;e.done?o(e.value):(t=e.value,t instanceof n?t:new n((function(e){e(t)}))).then(d,r)}u((i=i.apply(e,t||[])).next())}))};Object.defineProperty(t,"__esModule",{value:!0}),t.textToEmbeddings=void 0,t.textToEmbeddings=function({apiUrl:e="https://api.openai.com/v1/embeddings",apiKey:t,user:i,model:o="text-embedding-ada-002",text:a}){return n(this,void 0,void 0,(function*(){if(e.includes("api.openai.com")){const n={"Content-Type":"application/json",Authorization:`Bearer ${null!=t?t:""}`};let d;d=yield fetch(e,{headers:n,method:"POST",body:JSON.stringify({user:i,model:o,input:a})});const r=yield d.json();if(!r.data||0===r.data.length)return;return r.data[0].embedding}}))}}})[899](0,e),module.exports=e})(); \ No newline at end of file diff --git a/packages/text-to-embeddings/package.json b/packages/text-to-embeddings/package.json new file mode 100644 index 0000000..f7b2996 --- /dev/null +++ b/packages/text-to-embeddings/package.json @@ -0,0 +1,45 @@ +{ + "name": "text-to-embeddings", + "version": "0.0.0", + "description": "A utility to convert text to vector embeddings.", + "author": "Jasper Alexander", + "license": "MIT", + "main": "dist/index.js", + "files": [ + "dist/**/*" + ], + "scripts": { + "build": "webpack", + "lint": "eslint src/*.ts*" + }, + "publishConfig": { + "access": "public", + "registry": "https://registry.npmjs.org" + }, + "repository": { + "type": "git", + "url": "https://github.com/JasperAlexander/ai-utils.git" + }, + "keywords": [ + "ai", + "text", + "convert", + "converter", + "gpt" + ], + "bugs": { + "url": "https://github.com/JasperAlexander/ai-utils/issues" + }, + "homepage": "https://github.com/JasperAlexander/ai-utils/tree/main/packages/text-to-embeddings/README.md", + "devDependencies": { + "eslint": "^7.32.0", + "eslint-config-custom": "workspace:*", + "ts-loader": "^9.4.2", + "typescript": "^4.9.5", + "webpack": "^5.81.0", + "webpack-cli": "^5.0.2" + }, + "dependencies": { + "@types/node": "^17.0.45" + } +} diff --git a/packages/text-to-embeddings/src/index.ts b/packages/text-to-embeddings/src/index.ts new file mode 100644 index 0000000..541f2b0 --- /dev/null +++ b/packages/text-to-embeddings/src/index.ts @@ -0,0 +1,45 @@ +/** + * Converts text to vector embeddings, currently only supports OpenAI + * @param apiUrl The url of the embeddings API, defaults to https://api.openai.com/v1/embeddings + * @param apiKey The key of the API + * @param user A unique identifier representing the user + * @param model The model to use, defaults to text-embedding-ada-002 + * @param text The text to convert + * @returns The AI-generated text or ReadableStream + */ +export async function textToEmbeddings({ + apiUrl = 'https://api.openai.com/v1/embeddings', + apiKey, + user, + model = 'text-embedding-ada-002', + text +}: { + apiUrl?: string + apiKey?: string + user?: string + model?: string + text: string +}) { + if(apiUrl.includes('api.openai.com')) { + const requestHeaders: Record = { + 'Content-Type': 'application/json', + Authorization: `Bearer ${apiKey ?? ''}`, + } + + let response: Response + response = await fetch(apiUrl, { + headers: requestHeaders, + method: 'POST', + body: JSON.stringify({ + user, + model, + input: text, + }) + }) + + const body = await response.json() + + if(!body.data || body.data.length === 0) return + return body.data[0].embedding + } +} \ No newline at end of file diff --git a/packages/text-to-embeddings/tsconfig.json b/packages/text-to-embeddings/tsconfig.json new file mode 100644 index 0000000..5f467bc --- /dev/null +++ b/packages/text-to-embeddings/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "declaration": true, + "esModuleInterop": true, + "module": "CommonJS", + "moduleResolution": "node", + "outDir": "dist", + "rootDir": "src", + "strict": true, + "target": "ES6" + }, + "include": ["src"] +} diff --git a/packages/text-to-embeddings/webpack.config.js b/packages/text-to-embeddings/webpack.config.js new file mode 100644 index 0000000..6e374a4 --- /dev/null +++ b/packages/text-to-embeddings/webpack.config.js @@ -0,0 +1,24 @@ +const path = require('path') + +module.exports = { + entry: './src/index.ts', + mode: 'production', + target: 'node', + module: { + rules: [ + { + test: /\.tsx?$/, + use: 'ts-loader', + exclude: /node_modules/, + }, + ], + }, + resolve: { + extensions: ['.tsx', '.ts', '.js'], + }, + output: { + filename: 'index.js', + path: path.resolve(__dirname, 'dist'), + libraryTarget: 'commonjs2', + }, +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0dc32de..3cf20fd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -267,6 +267,31 @@ importers: specifier: ^5.0.2 version: 5.0.2(webpack@5.81.0) + packages/text-to-embeddings: + dependencies: + '@types/node': + specifier: ^17.0.45 + version: 17.0.45 + devDependencies: + eslint: + specifier: ^7.32.0 + version: 7.32.0 + eslint-config-custom: + specifier: workspace:* + version: link:../eslint-config-custom + ts-loader: + specifier: ^9.4.2 + version: 9.4.2(typescript@4.9.5)(webpack@5.81.0) + typescript: + specifier: ^4.9.5 + version: 4.9.5 + webpack: + specifier: ^5.81.0 + version: 5.81.0(webpack-cli@5.0.2) + webpack-cli: + specifier: ^5.0.2 + version: 5.0.2(webpack@5.81.0) + packages: /@ampproject/remapping@2.2.1: