diff --git a/.bioserver.env.template b/.bioserver.env.template index be16aff..d38f45f 100644 --- a/.bioserver.env.template +++ b/.bioserver.env.template @@ -1,5 +1,6 @@ # If .bioserver.env doesn't provide OPENAI setup, customers need to set OPENAI API key in website # OPENAI_API_KEY= +# OPENAI_MODEL= # Optional (Azure), # Besides OPENAI_API_KEY, the following envs should be set for Azure @@ -11,3 +12,9 @@ # AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME= # AZURE_OPENAI_EMBEDDINGS_MODEL= +# data directory for sqlite3 db +DATA_DIR=/app/data + +# daily token limit for server +TOKEN_DAILY_LIMITATION=1_000_000 + diff --git a/CHANGELOG.md b/CHANGELOG.md index b33f548..7fc2ba0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [1.4.0](https://github.com/fengsh27/chatgse-next-sync/compare/v1.3.5...v1.4.0) (2025-01-06) + + +### Features + +* display token usage ([#14](https://github.com/fengsh27/chatgse-next-sync/issues/14)) ([a9565a4](https://github.com/fengsh27/chatgse-next-sync/commit/a9565a449b852123728651a8bbe816e664ec9668)) + ### [1.3.5](https://github.com/biocypher/biochatter-next/compare/v1.3.4...v1.3.5) (2024-08-26) diff --git a/app/api/rag/connectionstatus/route.ts b/app/api/rag/connectionstatus/route.ts index f10dc0c..342d287 100644 --- a/app/api/rag/connectionstatus/route.ts +++ b/app/api/rag/connectionstatus/route.ts @@ -1,6 +1,6 @@ import { NextRequest, NextResponse } from "next/server"; -import { BioChatterServerResponse, getBaseUrl } from "../../common"; -import { BiochatterPath, ERROR_BIOSERVER_MILVUS_CONNECT_FAILED, ERROR_BIOSERVER_OK, ERROR_BIOSERVER_UNKNOWN } from "@/app/constant"; +import { getBaseUrl } from "../../common"; +import { BiochatterPath } from "@/app/constant"; import { prettyObject } from "@/app/utils/format"; async function handle(request: NextRequest) { diff --git a/app/api/tokenusage/route.ts b/app/api/tokenusage/route.ts new file mode 100644 index 0000000..36df2bc --- /dev/null +++ b/app/api/tokenusage/route.ts @@ -0,0 +1,37 @@ +import { NextRequest, NextResponse } from "next/server"; +import { getBaseUrl } from "../common"; +import { BiochatterPath} from "@/app/constant"; +import { prettyObject } from "@/app/utils/format"; + +async function handle(request: NextRequest) { + const AUTHORIZATION = "Authorization" + const authValue = request.headers.get(AUTHORIZATION) ?? ""; + const baseUrl = getBaseUrl(); + const data = await request.json(); + const path = BiochatterPath.TokenUsage; + const url = `${baseUrl}/${path}`; + try { + const res = await fetch( + url, + { + method: "POST", + headers: { + [AUTHORIZATION]: authValue, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + model: data.model ?? "gpt-3.5-turbo", + session_id: data.session_id ?? "", + }), + } + ); + const jsonBody = await res.json(); + return NextResponse.json(jsonBody); + } catch (e: any) { + console.error(e); + return NextResponse.json(prettyObject(e)); + } +} + +export const POST = handle; + diff --git a/app/client/api.ts b/app/client/api.ts index 7b7b669..1e38398 100644 --- a/app/client/api.ts +++ b/app/client/api.ts @@ -8,7 +8,7 @@ import { ChatGPTApi } from "./platforms/openai"; export const ROLES = ["system", "user", "assistant"] as const; export type MessageRole = (typeof ROLES)[number]; -export const Models = ["gpt-3.5-turbo", "gpt-4"] as const; +export const Models = ["gpt-3.5-turbo", "gpt-4", "gpt-4o"] as const; export type ChatModel = ModelType; export interface RequestMessage { diff --git a/app/client/datarequest.ts b/app/client/datarequest.ts index 00048c2..a668960 100644 --- a/app/client/datarequest.ts +++ b/app/client/datarequest.ts @@ -121,4 +121,26 @@ export const requestRemoveDocument = async ( } }); return res; +} + +export const requestTokenUsage = async ( + session_id?: string, + model?: string +) => { + const TokenUsage_URL = ApiPath.TokenUsage; + let url = TokenUsage_URL as string; + if (!url.endsWith('/')) { + url += '/'; + } + const res = await fetch(url, { + method: "POST", + body: JSON.stringify({ + "session_id": session_id ?? "", + "model": model ?? "gpt-3.5-turbo", + }), + headers: { + ...get_auth_header() + } + }); + return res; } \ No newline at end of file diff --git a/app/client/platforms/openai.ts b/app/client/platforms/openai.ts index 221e1d2..d2a082b 100644 --- a/app/client/platforms/openai.ts +++ b/app/client/platforms/openai.ts @@ -2,11 +2,12 @@ import { ApiPath, DEFAULT_API_HOST, DEFAULT_MODELS, + ERROR_BIOSERVER_EXCEEDS_TOKEN_LIMIT, OpenaiPath, REQUEST_TIMEOUT_MS, ServiceProvider, } from "@/app/constant"; -import { ChatSession, useAccessStore, useAppConfig, useChatStore } from "@/app/store"; +import { useAccessStore, useAppConfig, useChatStore } from "@/app/store"; import { ChatOptions, getHeaders, LLMApi, LLMModel, LLMUsage } from "../api"; import Locale from "../../locales"; @@ -17,9 +18,6 @@ import { import { prettyObject } from "@/app/utils/format"; import { getClientConfig } from "@/app/config/client"; import { makeAzurePath } from "@/app/azure"; -import { useRAGStore } from "@/app/store/rag"; -import { useKGStore } from "@/app/store/kg"; -import { getKnowledgeGraphInfo, getOncoKBInfo, getVectorStoreInfo } from "@/app/utils/prodinfo"; export interface OpenAIListModelResponse { object: string; @@ -236,6 +234,10 @@ export class ChatGPTApi implements LLMApi { clearTimeout(requestTimeoutId); const resJson = await res.json(); + if (resJson.code === ERROR_BIOSERVER_EXCEEDS_TOKEN_LIMIT && options.onFinish) { + options.onFinish(Locale.Chat.TokenLimit(resJson.limitation)); + return; + } const message = this.extractMessage(resJson); options.onFinish(message, resJson.contexts); } diff --git a/app/components/auth.tsx b/app/components/auth.tsx index 7962d46..372d291 100644 --- a/app/components/auth.tsx +++ b/app/components/auth.tsx @@ -42,31 +42,15 @@ export function AuthPage() { { accessStore.update( - (access) => (access.accessCode = e.currentTarget.value), + (access) => (access.openaiApiKey = e.currentTarget.value), ); }} /> - {!accessStore.hideUserApiKey ? ( - <> -
{Locale.Auth.SubTips}
- { - accessStore.update( - (access) => (access.openaiApiKey = e.currentTarget.value), - ); - }} - /> - - ) : null} - +
- + {false && ( {showAccessCode && ( + )} { + accessStore.updateTokenUsage(sessionId, currentModel); + }, []); + // switch themes function nextTheme() { const themes = [Theme.Auto, Theme.Light, Theme.Dark]; @@ -237,7 +243,10 @@ export function SideBar(props: { className?: string }) {
-
+
+
} @@ -258,6 +267,17 @@ export function SideBar(props: { className?: string }) { }} shadow /> +
+ {!shouldNarrow && ( +
+ {useAccessStore.getState().tokenUsage.auth_type.slice(0, 6) === "Server" ? ( +
Server token usage
+ ) : ( +
Client token usage
+ )} +
Total tokens: {useAccessStore.getState().tokenUsage.tokens.total_tokens}
+
+ )}
diff --git a/app/constant.ts b/app/constant.ts index bf609ae..6441e8d 100644 --- a/app/constant.ts +++ b/app/constant.ts @@ -32,6 +32,7 @@ export enum ApiPath { OpenAI = "/api/openai", RAG = "/api/rag", KG = "/api/kg", + TokenUsage = "/api/tokenusage" } export enum SlotID { @@ -91,6 +92,7 @@ export const BiochatterPath = { Document: "v1/rag/document", RAGConnectionStatus: "v1/rag/connectionstatus", KGConnectionStatus: "v1/kg/connectionstatus", + TokenUsage: "v1/tokenusage", } export const Azure = { @@ -115,21 +117,26 @@ export const KnowledgeCutOffDate: Record = { "gpt-4-vision-preview": "2023-04", }; -export const DEFAULT_MODELS = [ - { - name: "gpt-4o", - available: true, - }, -] as const; +export const DEFAULT_MODELS = [{ + name: "gpt-4o", + available: true, +}, { + name: "gpt-4", + available: true, +}, { + name: "gpt-3.5-turbo", + available: true, +}] as const; export const CHAT_PAGE_SIZE = 15; export const MAX_RENDER_MSG_COUNT = 45; // biochatter-server error export const ERROR_BIOSERVER_OK = 0 -export const ERROR_BIOSERVER_UNKNOWN = 5000 +export const ERROR_BIOSERVER_UNKNOWN = 5100 export const ERROR_BIOSERVER_MILVUS_UNKNOWN = 5101 export const ERROR_BIOSERVER_MILVUS_CONNECT_FAILED = 5102 +export const ERROR_BIOSERVER_EXCEEDS_TOKEN_LIMIT = 5103 export const HDR_CONTENT_TYPE = "Content-Type"; export const HDR_APPLICATION_JSON = "application/json"; diff --git a/app/locales/cn.ts b/app/locales/cn.ts index ffd8247..e7f1877 100644 --- a/app/locales/cn.ts +++ b/app/locales/cn.ts @@ -11,8 +11,8 @@ const cn = { : "访问密码不正确或为空，请前往[登录](/#/auth)页输入正确的访问密码，或者在[设置](/#/settings)页填入你自己的 OpenAI API Key。", }, Auth: { - Title: "需要密码", - Tips: "管理员开启了密码验证，请在下方填入访问码", + Title: "需要OpenAI API Key", + Tips: "请在下方填入OpenAI API Key", SubTips: "或者输入你的 OpenAI API 密钥", Input: "在此处填写访问码", Confirm: "确认", @@ -119,6 +119,7 @@ const cn = { RAG: "", KG: "", }, + TokenLimit: (x: any) => `The server's community daily token limit (${x} tokens) has been reached. \n\n Please provide your OpenAI API key on the [auth](/#/auth) page to continue.`, }, Export: { Title: "分享聊天记录", diff --git a/app/locales/en.ts b/app/locales/en.ts index 0ca3f29..f21e8ea 100644 --- a/app/locales/en.ts +++ b/app/locales/en.ts @@ -13,8 +13,8 @@ const en: LocaleType = { : "Unauthorized access, please enter access code in [auth](/#/auth) page, or enter your OpenAI API Key.", }, Auth: { - Title: "Need Access Code", - Tips: "Please enter access code below", + Title: "Need OpenAI API Key", + Tips: "Please enter your OpenaAI API Key below", SubTips: "Or enter your OpenAI API Key", Input: "access code", Confirm: "Confirm", @@ -121,6 +121,7 @@ const en: LocaleType = { RAG: "Calling RAG", KG: "Calling KG RAG", }, + TokenLimit: (x: any) => `The server's community daily token limit (${x} tokens) has been reached. \n\n Please provide your OpenAI API key on the [auth](/#/auth) page to continue.`, }, Export: { Title: "Export Messages", diff --git a/app/store/access.ts b/app/store/access.ts index be7706f..7191970 100644 --- a/app/store/access.ts +++ b/app/store/access.ts @@ -8,6 +8,7 @@ import { getHeaders } from "../client/api"; import { getClientConfig } from "../config/client"; import { createPersistStore } from "../utils/store"; import { ensure } from "../utils/clone"; +import { requestTokenUsage } from "../client/datarequest"; let fetchState = 0; // 0 not fetch, 1 fetching, 2 done @@ -37,6 +38,14 @@ const DEFAULT_ACCESS_STATE = { disableFastLink: false, customModels: "", productionInfo: "undefined", + tokenUsage: { + auth_type: "Unknown", + tokens: { + "completion_tokens": 0, + "prompt_tokens": 0, + "total_tokens": 0, + } + } }; export const useAccessStore = createPersistStore( @@ -88,10 +97,26 @@ export const useAccessStore = createPersistStore( fetchState = 2; } }, + async updateTokenUsage(session_id: string, model: string) { + requestTokenUsage(session_id, model).then((res: any) => { + res.json().then((dat: any) => { + set({ + tokenUsage: { + auth_type: dat.auth_type ?? "Unknown", + tokens: { + completion_tokens: dat.tokens?.completion_tokens ?? 0, + prompt_tokens: dat.tokens?.prompt_tokens ?? 0, + total_tokens: dat.tokens?.total_tokens ?? 0, + } + } + }); + }); + }); + }, }), { name: StoreKey.Access, - version: 2.1, + version: 2.2, migrate(persistedState, version) { if (version < 2) { const state = persistedState as { @@ -108,6 +133,24 @@ export const useAccessStore = createPersistStore( } state.productionInfo = "undefined"; } + if (version < 2.2) { + const state = persistedState as { + tokenUsage: { + auth_type: string, + tokens: { + "completion_tokens": number, + "prompt_tokens": number, + "total_tokens": number, + } + } + } + state.tokenUsage.auth_type = "Unknown"; + state.tokenUsage.tokens = { + "completion_tokens": 0, + "prompt_tokens": 0, + "total_tokens": 0, + } + } return persistedState as any; }, diff --git a/app/store/chat.ts b/app/store/chat.ts index ac6c23a..24f6bd0 100644 --- a/app/store/chat.ts +++ b/app/store/chat.ts @@ -192,6 +192,14 @@ export const useChatStore = createPersistStore( set({ currentSessionIndex: index, }); + + // on session changed + if (index >= get().sessions.length) { + return; + } + const model = get().sessions[index].mask?.modelConfig?.model ?? "gpt-3.5-turbo"; + const sessionId = get().sessions[index].id; + useAccessStore.getState().updateTokenUsage(sessionId, model); }, moveSession(from: number, to: number) { @@ -407,6 +415,7 @@ export const useChatStore = createPersistStore( get().onNewMessage(botMessage); } ChatControllerPool.remove(session.id, botMessage.id); + useAccessStore.getState().updateTokenUsage(session.id, modelConfig.model); }, onError(error) { const isAborted = error.message.includes("aborted"); diff --git a/app/utils/prodinfo.ts b/app/utils/prodinfo.ts index ee9c7f0..2e16f8b 100644 --- a/app/utils/prodinfo.ts +++ b/app/utils/prodinfo.ts @@ -10,7 +10,7 @@ import Locale from "../locales"; import { LLMModel } from "../client/api"; export function getOncoKBInfo(prodInfo?: ProductionInfo): APIAgentInfo { - return (prodInfo?.OncoKBAPI ?? {enabled: true}) + return (prodInfo?.OncoKBAPI ?? {enabled: false}) } export function getKnowledgeGraphInfo(prodInfo?: ProductionInfo): DbConfiguration { diff --git a/app/utils/rag.ts b/app/utils/rag.ts index d2b50b7..eaf7504 100644 --- a/app/utils/rag.ts +++ b/app/utils/rag.ts @@ -9,7 +9,7 @@ const getConnectionArgsToDisplay = ( if (server.server === connectionArgs.host) { return { host: server.server, - port: `${server.port}` ?? defaultPort, + port: `${server.port ?? defaultPort}`, } } const serverPort = `${server.port??defaultPort}`; @@ -113,4 +113,4 @@ export const getServerDescription = ( } } return undefined; -} \ No newline at end of file +} diff --git a/package.json b/package.json index bae1a52..1783350 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "biochatter-next", "private": false, "license": "MIT", - "version": "1.3.5", + "version": "1.4.0", "scripts": { "dev": "next dev", "build": "cross-env BUILD_MODE=standalone next build", diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 7592061..7269d39 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -9,7 +9,7 @@ }, "package": { "productName": "BioChatter-Next", - "version": "1.3.5" + "version": "1.4.0" }, "tauri": { "allowlist": {