Skip to content

Commit

Permalink
connect realtime model when open panel
Browse files Browse the repository at this point in the history
  • Loading branch information
@lloydzhou
lloydzhou committed Nov 7, 2024
1 parent 88cd3ac commit 7bdc45e
Show file tree
Hide file tree
Showing 2 changed files with 41 additions and 236 deletions.
18 changes: 10 additions & 8 deletions app/components/chat.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2035,14 +2035,16 @@ function _Chat() {
[styles["chat-side-panel-show"]]: showChatSidePanel,
})}
>
<RealtimeChat
onClose={() => {
setShowChatSidePanel(false);
}}
onStartVoice={async () => {
console.log("start voice");
}}
/>
{showChatSidePanel && (
<RealtimeChat
onClose={() => {
setShowChatSidePanel(false);
}}
onStartVoice={async () => {
console.log("start voice");
}}
/>
)}
</div>
</div>
</div>
Expand Down
259 changes: 31 additions & 228 deletions app/components/realtime-chat/realtime-chat.tsx
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { useDebouncedCallback } from "use-debounce";
import VoiceIcon from "@/app/icons/voice.svg";
import VoiceOffIcon from "@/app/icons/voice-off.svg";
import Close24Icon from "@/app/icons/close-24.svg";
import PowerIcon from "@/app/icons/power.svg";

import styles from "./realtime-chat.module.scss";
Expand Down Expand Up @@ -60,6 +60,7 @@ export function RealtimeChat({
const apiKey = accessStore.openaiApiKey;

const handleConnect = async () => {
if (isConnecting) return;
if (!isConnected) {
try {
setIsConnecting(true);
Expand Down Expand Up @@ -230,215 +231,34 @@ export function RealtimeChat({
}
};

useEffect(() => {
const initAudioHandler = async () => {
const handler = new AudioHandler();
await handler.initialize();
audioHandlerRef.current = handler;
};
useEffect(
useDebouncedCallback(() => {
const initAudioHandler = async () => {
const handler = new AudioHandler();
await handler.initialize();
audioHandlerRef.current = handler;
await handleConnect();
await toggleRecording();
};

initAudioHandler().catch(console.error);
initAudioHandler().catch(console.error);

return () => {
disconnect();
audioHandlerRef.current?.close().catch(console.error);
};
}, []);

// useEffect(() => {
// if (
// clientRef.current?.getTurnDetectionType() === "server_vad" &&
// audioData
// ) {
// // console.log("appendInputAudio", audioData);
// // 将录制的16PCM音频发送给openai
// clientRef.current?.appendInputAudio(audioData);
// }
// }, [audioData]);

// useEffect(() => {
// console.log("isRecording", isRecording);
// if (!isRecording.current) return;
// if (!clientRef.current) {
// const apiKey = accessStore.openaiApiKey;
// const client = (clientRef.current = new RealtimeClient({
// url: "wss://api.openai.com/v1/realtime",
// apiKey,
// dangerouslyAllowAPIKeyInBrowser: true,
// debug: true,
// }));
// client
// .connect()
// .then(() => {
// // TODO 设置真实的上下文
// client.sendUserMessageContent([
// {
// type: `input_text`,
// text: `Hi`,
// // text: `For testing purposes, I want you to list ten car brands. Number each item, e.g. "one (or whatever number you are one): the item name".`
// },
// ]);

// // 配置服务端判断说话人开启还是结束
// client.updateSession({
// turn_detection: { type: "server_vad" },
// });

// client.on("realtime.event", (realtimeEvent) => {
// // 调试
// console.log("realtime.event", realtimeEvent);
// });

// client.on("conversation.interrupted", async () => {
// if (currentBotMessage.current) {
// stopPlaying();
// try {
// client.cancelResponse(
// currentBotMessage.current?.id,
// currentTime(),
// );
// } catch (e) {
// console.error(e);
// }
// }
// });
// client.on("conversation.updated", async (event: any) => {
// // console.log("currentSession", chatStore.currentSession());
// // const items = client.conversation.getItems();
// const content = event?.item?.content?.[0]?.transcript || "";
// const text = event?.item?.content?.[0]?.text || "";
// // console.log(
// // "conversation.updated",
// // event,
// // "content[0]",
// // event?.item?.content?.[0]?.transcript,
// // "formatted",
// // event?.item?.formatted?.transcript,
// // "content",
// // content,
// // "text",
// // text,
// // event?.item?.status,
// // event?.item?.role,
// // items.length,
// // items,
// // );
// const { item, delta } = event;
// const { role, id, status, formatted } = item || {};
// if (id && role == "assistant") {
// if (
// !currentBotMessage.current ||
// currentBotMessage.current?.id != id
// ) {
// // create assistant message and save to session
// currentBotMessage.current = createMessage({ id, role });
// chatStore.updateCurrentSession((session) => {
// session.messages = session.messages.concat([
// currentBotMessage.current!,
// ]);
// });
// }
// if (currentBotMessage.current?.id != id) {
// stopPlaying();
// }
// if (content) {
// currentBotMessage.current.content = content;
// chatStore.updateCurrentSession((session) => {
// session.messages = session.messages.concat();
// });
// }
// if (delta?.audio) {
// // typeof delta.audio is Int16Array
// // 直接播放
// addInt16PCM(delta.audio);
// }
// // console.log(
// // "updated try save wavFile",
// // status,
// // currentBotMessage.current?.audio_url,
// // formatted?.audio,
// // );
// if (
// status == "completed" &&
// !currentBotMessage.current?.audio_url &&
// formatted?.audio?.length
// ) {
// // 转换为wav文件保存 TODO 使用mp3格式会更节省空间
// const botMessage = currentBotMessage.current;
// const wavFile = new WavPacker().pack(sampleRate, {
// bitsPerSample: 16,
// channelCount: 1,
// data: formatted?.audio,
// });
// // 这里将音频文件放到对象里面wavFile.url可以使用<audio>标签播放
// item.formatted.file = wavFile;
// uploadImageRemote(wavFile.blob).then((audio_url) => {
// botMessage.audio_url = audio_url;
// chatStore.updateCurrentSession((session) => {
// session.messages = session.messages.concat();
// });
// });
// }
// if (
// status == "completed" &&
// !currentBotMessage.current?.content
// ) {
// chatStore.updateCurrentSession((session) => {
// session.messages = session.messages.filter(
// (m) => m.id !== currentBotMessage.current?.id,
// );
// });
// }
// }
// if (id && role == "user" && !text) {
// if (
// !currentUserMessage.current ||
// currentUserMessage.current?.id != id
// ) {
// // create assistant message and save to session
// currentUserMessage.current = createMessage({ id, role });
// chatStore.updateCurrentSession((session) => {
// session.messages = session.messages.concat([
// currentUserMessage.current!,
// ]);
// });
// }
// if (content) {
// // 转换为wav文件保存 TODO 使用mp3格式会更节省空间
// const userMessage = currentUserMessage.current;
// const wavFile = new WavPacker().pack(sampleRate, {
// bitsPerSample: 16,
// channelCount: 1,
// data: formatted?.audio,
// });
// // 这里将音频文件放到对象里面wavFile.url可以使用<audio>标签播放
// item.formatted.file = wavFile;
// uploadImageRemote(wavFile.blob).then((audio_url) => {
// // update message content
// userMessage.content = content;
// // update message audio_url
// userMessage.audio_url = audio_url;
// chatStore.updateCurrentSession((session) => {
// session.messages = session.messages.concat();
// });
// });
// }
// }
// });
// })
// .catch((e) => {
// console.error("Error", e);
// });
// }
// return () => {
// stop();
// // TODO close client
// clientRef.current?.disconnect();
// };
// }, [isRecording.current]);
return () => {
if (isRecording) {
toggleRecording();
}
audioHandlerRef.current?.close().catch(console.error);
disconnect();
};
}),
[],
);

const handleClose = () => {
const handleClose = async () => {
onClose?.();
if (isRecording) {
toggleRecording();
}
disconnect();
};

Expand All @@ -454,35 +274,18 @@ export function RealtimeChat({
<div className={styles["bottom-icons"]}>
<div>
<IconButton
icon={isRecording ? <VoiceOffIcon /> : <VoiceIcon />}
icon={!isRecording ? <VoiceOffIcon /> : <VoiceIcon />}
onClick={toggleRecording}
disabled={!isConnected}
bordered
shadow
type={isConnecting || isConnected ? "danger" : "primary"}
/>
</div>
<div className={styles["icon-center"]}>
<div className={styles["icon-center"]}></div>
<div>
<IconButton
icon={<PowerIcon />}
text={
isConnecting
? "Connecting..."
: isConnected
? "Disconnect"
: "Connect"
}
onClick={handleConnect}
disabled={isConnecting}
bordered
shadow
/>
</div>
<div onClick={handleClose}>
<IconButton
icon={<Close24Icon />}
onClick={handleClose}
bordered
shadow
type={isConnecting || isConnected ? "danger" : "primary"}
/>
</div>
</div>
Expand Down

0 comments on commit 7bdc45e

Please sign in to comment.