Added package text-to-embeddings

.changeset/tricky-dragons-hang.md

@@ -0,0 +1,5 @@
+---
+"text-to-embeddings": major
+---
+
+Added package text-to-embeddings

packages/text-to-embeddings/README.md

@@ -0,0 +1,44 @@
+# text-to-embeddings
+
+A utility to convert text to vector embeddings. Currently only supports OpenAI embeddings. The utility is made to be used on a Node.js server for security reasons.
+
+## Installation
+
+Go to the root of your project directory and run the following command.
+
+```sh
+pnpm add text-to-embeddings
+```
+
+## Usage
+
+Use text-to-embeddings in a Node.js server. In Next.js 13, you can make a route handler with the following code.
+
+```tsx
+import { NextResponse } from 'next/server'
+import { textToEmbeddings } from 'text-to-embeddings'
+
+export const config = {
+  runtime: 'edge',
+}
+
+export async function POST(req: Request) {
+  const body = await req.json()
+
+  const embeddings = await textToEmbeddings({
+    apiKey: process.env.OPENAI_API_KEY,
+    user: body.user,
+    text: body.text
+  })
+
+  return new Response(embeddings)
+}
+```
+
+## Contributing
+
+If you're interested in contributing, please read the [contributing docs](../../CONTRIBUTING.md) before submitting a pull request.
+
+## License
+
+This package is licensed under the MIT License. See [LICENSE](../../LICENSE.md) for more information.

packages/text-to-embeddings/dist/index.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Converts text to vector embeddings, currently only supports OpenAI
+ * @param apiUrl The url of the embeddings API, defaults to https://api.openai.com/v1/embeddings
+ * @param apiKey The key of the API
+ * @param user A unique identifier representing the user
+ * @param model The model to use, defaults to text-embedding-ada-002
+ * @param text The text to convert
+ * @returns The AI-generated text or ReadableStream
+ */
+export declare function textToEmbeddings({ apiUrl, apiKey, user, model, text }: {
+    apiUrl?: string;
+    apiKey?: string;
+    user?: string;
+    model?: string;
+    text: string;
+}): Promise<any>;

packages/text-to-embeddings/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "text-to-embeddings",
+  "version": "0.0.0",
+  "description": "A utility to convert text to vector embeddings.",
+  "author": "Jasper Alexander",
+  "license": "MIT",
+  "main": "dist/index.js",
+  "files": [
+    "dist/**/*"
+  ],
+  "scripts": {
+    "build": "webpack",
+    "lint": "eslint src/*.ts*"
+  },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/JasperAlexander/ai-utils.git"
+  },
+  "keywords": [
+    "ai",
+    "text",
+    "convert",
+    "converter",
+    "gpt"
+  ],
+  "bugs": {
+    "url": "https://github.com/JasperAlexander/ai-utils/issues"
+  },
+  "homepage": "https://github.com/JasperAlexander/ai-utils/tree/main/packages/text-to-embeddings/README.md",
+  "devDependencies": {
+    "eslint": "^7.32.0",
+    "eslint-config-custom": "workspace:*",
+    "ts-loader": "^9.4.2",
+    "typescript": "^4.9.5",
+    "webpack": "^5.81.0",
+    "webpack-cli": "^5.0.2"
+  },
+  "dependencies": {
+    "@types/node": "^17.0.45"
+  }
+}

packages/text-to-embeddings/src/index.ts

@@ -0,0 +1,45 @@
+/**
+ * Converts text to vector embeddings, currently only supports OpenAI
+ * @param apiUrl The url of the embeddings API, defaults to https://api.openai.com/v1/embeddings
+ * @param apiKey The key of the API
+ * @param user A unique identifier representing the user
+ * @param model The model to use, defaults to text-embedding-ada-002
+ * @param text The text to convert
+ * @returns The AI-generated text or ReadableStream
+ */
+export async function textToEmbeddings({
+  apiUrl = 'https://api.openai.com/v1/embeddings',
+  apiKey,
+  user,
+  model = 'text-embedding-ada-002',
+  text
+}: {
+  apiUrl?: string
+  apiKey?: string
+  user?: string
+  model?: string
+  text: string
+}) {
+  if(apiUrl.includes('api.openai.com')) {
+    const requestHeaders: Record<string, string> = {
+      'Content-Type': 'application/json',
+      Authorization: `Bearer ${apiKey ?? ''}`,
+    }
+
+    let response: Response
+    response = await fetch(apiUrl, {
+      headers: requestHeaders,
+      method: 'POST',
+      body: JSON.stringify({
+        user,
+        model,
+        input: text,
+      })
+    })
+
+    const body = await response.json()
+
+    if(!body.data || body.data.length === 0) return
+    return body.data[0].embedding
+  }
+}

packages/text-to-embeddings/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "declaration": true,
+    "esModuleInterop": true,
+    "module": "CommonJS",
+    "moduleResolution": "node",
+    "outDir": "dist",
+    "rootDir": "src",
+    "strict": true,
+    "target": "ES6"
+  },
+  "include": ["src"]
+}

packages/text-to-embeddings/webpack.config.js

@@ -0,0 +1,24 @@
+const path = require('path')
+
+module.exports = {
+  entry: './src/index.ts',
+  mode: 'production',
+  target: 'node',
+  module: {
+    rules: [
+      {
+        test: /\.tsx?$/,
+        use: 'ts-loader',
+        exclude: /node_modules/,
+      },
+    ],
+  },
+  resolve: {
+    extensions: ['.tsx', '.ts', '.js'],
+  },
+  output: {
+    filename: 'index.js',
+    path: path.resolve(__dirname, 'dist'),
+    libraryTarget: 'commonjs2',
+  },
+}