bumped avatar-3d version, fixed audiocontext bug for avatar3d, added …

…avatar3d component props to CharacterAvatar
AIPL-labs · Nov 8, 2024 · 294eced · 294eced
1 parent 90fc1a4
commit 294eced
Show file tree

Hide file tree

Showing 5 changed files with 103 additions and 23 deletions.
diff --git a/bun.lockb b/bun.lockb
diff --git a/package.json b/package.json
@@ -65,15 +65,13 @@
     "@storybook/test": "^8.2.9",
     "storybook": "^8.2.9"
   },
-  "peerDependencies": {
-  },
+  "peerDependencies": {},
   "dependencies": {
     "@emotion/react": "^11.13.3",
     "@emotion/styled": "^11.13.0",
     "@lunapaint/png-codec": "^0.2.0",
+    "@mjtdev/avatar-3d": "https://github.com/matthewjosephtaylor/avatar-3d#2024.11.8-1700",
     "@mjtdev/engine": "https://github.com/matthewjosephtaylor/mjtdev-engine#2024.10.25-1350",
-    "ai-worker-common": "https://github.com/AIPL-labs/ai-worker-common#2024.10.28-1208",
-    "@mjtdev/avatar-3d": "https://github.com/matthewjosephtaylor/avatar-3d#2024.10.28-0933",
     "@monaco-editor/react": "^4.6.0",
     "@mui/material": "^6.0.1",
     "@radix-ui/react-collapsible": "^1.1.1",
@@ -82,6 +80,7 @@
     "@radix-ui/themes": "^3.1.3",
     "@ricky0123/vad-web": "^0.0.18",
     "@sinclair/typebox": "^0.33.7",
+    "ai-worker-common": "https://github.com/AIPL-labs/ai-worker-common#2024.10.28-1208",
     "babel-plugin-styled-components": "^2.1.4",
     "character-card-utils": "^2.0.3",
     "easing-utils": "^1.0.0",

diff --git a/src/ui/character/CharacterAvatar.tsx b/src/ui/character/CharacterAvatar.tsx
@@ -1,13 +1,27 @@
 import { Avatar3dGltf } from "@mjtdev/avatar-3d";
-import { Colors, Objects, isDefined, type ByteLike } from "@mjtdev/engine";
-import { Badge, Card, Flex, Separator, Text } from "@radix-ui/themes";
+import {
+  Animates,
+  Colors,
+  Maths,
+  Objects,
+  isDefined,
+  type ArcRotateCameraOptions,
+  type ByteLike,
+} from "@mjtdev/engine";
+import {
+  Badge,
+  Card,
+  CardProps,
+  Flex,
+  Separator,
+  Text,
+} from "@radix-ui/themes";
 import type { AppCharacter } from "ai-worker-common";
 import { AnimatePresence, motion } from "framer-motion";
 import type { CSSProperties } from "react";
-import { memo, useState } from "react";
-import { useTtsState } from "../../tts/TtsState";
+import { memo, useEffect, useState } from "react";
+import { getTtsState, useTtsState } from "../../tts/TtsState";
 import { DEFAULT_CHAR_URL } from "../DEFAULT_CHAR_URL";
-import { stringifyEq } from "../chat/stringifyEq";
 import { AppButtonGroup } from "../common/AppButtonGroup";
 import { AppContextMenu } from "../common/AppContextMenu";
 import { CorpusDocumentListButton } from "../corpus/CorpusDocumentListButton";
@@ -17,6 +31,21 @@ import { idToColor } from "../visual/idToColor";
 import { VideoPlayer } from "./VideoPlayer";
 import type { CharacterAction } from "./characterToActions";
 import { characterToActions } from "./characterToActions";
+import { AudioContextVisualization } from "../chat/AudioContextVisualization";
+import { Ttss } from "../../tts/Ttss";
+// import { stringifyEq } from "../chat/stringifyEq";
+
+export const stringifyEq2 = (a: any, b: any) => {
+  if (a instanceof AnalyserNode || b instanceof AnalyserNode) {
+    return a === b;
+  }
+
+  const result = JSON.stringify(a) === JSON.stringify(b);
+  // if (!result) {
+  //   console.log("stringifyEq", { a, b });
+  // }
+  return result;
+};
 
 export const CharacterAvatar = memo(
   ({
@@ -36,7 +65,12 @@ export const CharacterAvatar = memo(
     hoverActions,
     buttonActions = [],
     onClick = () => {},
-  }: {
+    analyserNode,
+    avatar3dCanvasWidth,
+    avatar3dCanvasHeight,
+    avatar3dCameraOptions,
+    ...rest
+  }: CardProps & {
     video?: ByteLike;
     enableDocumentDrop?: boolean;
     showHoverButtons?: boolean;
@@ -48,15 +82,18 @@ export const CharacterAvatar = memo(
     showNotes?: boolean;
     showContextMenu?: boolean;
     show3dAvatar?: boolean;
+    avatar3dCanvasWidth?: number;
+    avatar3dCanvasHeight?: number;
     imageStyle?: CSSProperties;
     nameStyle?: CSSProperties;
     style?: CSSProperties;
     character: AppCharacter | undefined;
     onClick?: (characterId: string) => void;
+    analyserNode?: AnalyserNode;
+    avatar3dCameraOptions?: ArcRotateCameraOptions;
   }) => {
     const [pointerOver, setPointerOver] = useState(false);
     const [videoEnded, setVideoEnded] = useState(false);
-    const { analyserNode } = useTtsState();
 
     const backgroundColor = Colors.from("black").alpha(0.5).toString();
     const nameDisplay = showName ? (
@@ -66,6 +103,33 @@ export const CharacterAvatar = memo(
       return <></>;
     }
 
+    useEffect(() => {
+      if (!character?.card?.data?.extensions?.avatar3dUrl) {
+        return;
+      }
+      // if (currentSource && analyserNode) {
+      //   currentSource.connect(analyserNode);
+      // }
+
+      const ac = Animates.create({
+        ticker: () => {
+          const currentSource = getTtsState().currentSource;
+          if (currentSource && analyserNode) {
+            currentSource.connect(analyserNode);
+          }
+        },
+      });
+
+      // Ttss.toggleTts().then(async () => {
+      //   console.log("TTS toggled to regain audiocontext");
+      //   await Ttss.toggleTts();
+      //   console.log("TTS toggled back");
+      // });
+      return () => {
+        ac.destroy();
+      };
+    }, [analyserNode, character]);
+
     const tags = character?.card?.data?.tags ?? [];
 
     const tagColors = tags.map((tag) => {
@@ -141,6 +205,7 @@ export const CharacterAvatar = memo(
             whiteSpace: "nowrap",
             ...style,
           }}
+          {...rest}
         >
           <Flex gap="2" direction={"column"}>
             <Flex>
@@ -169,10 +234,19 @@ export const CharacterAvatar = memo(
                   >
                     {show3dAvatar &&
                     character.card.data.extensions?.avatar3dUrl ? (
-                      <Flex style={{ maxHeight: "40vh", overflow: "auto" }}>
+                      <Flex
+                        style={{
+                          maxHeight: "40vh",
+                          overflow: "auto",
+                          ...style,
+                        }}
+                      >
                         <Avatar3dGltf
                           path={character.card.data.extensions.avatar3dUrl}
                           analyserNode={analyserNode}
+                          canvasWidth={avatar3dCanvasWidth}
+                          canvasHeight={avatar3dCanvasHeight}
+                          cameraOptions={avatar3dCameraOptions}
                           // showPhonemes={true}
                           // showControls={true}
                         />
@@ -256,5 +330,5 @@ export const CharacterAvatar = memo(
       </AppContextMenu>
     );
   },
-  stringifyEq
+  stringifyEq2
 );
diff --git a/src/ui/chat/ChatBox.tsx b/src/ui/chat/ChatBox.tsx
@@ -46,8 +46,9 @@ export const ChatBox = memo(
     const [windowHeight, setWindowHeight] = useState(0);
     const { modes } = AppModes.useAppModesAndParams();
     // const { audioContext: ttsAudioContext, analyserNode } = getTtsState();
-    const { analyserNode } = getTtsState();
+    const { audioContext } = getTtsState();
     const { appearance } = useAppState();
+    const ttsAnalyzer = audioContext?.createAnalyser();
 
     const ttsSpeaking = useIsTtsSpeaking();
 
@@ -111,6 +112,11 @@ export const ChatBox = memo(
         character={aiCharacter}
         show3dAvatar={true}
         video={greetingVideo}
+        analyserNode={ttsAnalyzer}
+        // avatar3dCanvasWidth={100}
+        // avatar3dCanvasHeight={500}
+        // avatar3dCameraOptions={{ radius: 2 }}
+        // style={{ height: "50vh", width: "30vw" }}
       />
     ) : undefined;
     return (
@@ -187,7 +193,8 @@ export const ChatBox = memo(
                   // TODO how to get TTS audio context using react state?
                   // audioContext={audioPlayer.getAudioContext()}
                   // audioContext={ttsAudioContext}
-                  analyzerNode={analyserNode}
+                  // analyzerNode={analyserNode}
+                  analyzerNode={ttsAnalyzer}
                   color={audioVisualizationColor}
                   getSource={() => getTtsState().currentSource}
                   key={`${chat.id}-tts-visualization-${ttsSpeaking}`}

diff --git a/vite.config.ts b/vite.config.ts
@@ -46,15 +46,15 @@ export default defineConfig({
     // __APP_FRONT__: JSON.stringify("power-user"),
 
     // AI-UI Production settings
-    __HOME_BASE__: JSON.stringify("https://ai-worker.intelligage.workers.dev"),
-    __PAP_ID__: JSON.stringify(
-      "access-point-1725571260862-23455f95-1253-4dfe-96e3-952ac6af647c"
-    ),
-    __APP_FRONT__: JSON.stringify("pizza-demo"),
+    // __HOME_BASE__: JSON.stringify("https://ai-worker.intelligage.workers.dev"),
+    // __PAP_ID__: JSON.stringify(
+    //   "access-point-1725571260862-23455f95-1253-4dfe-96e3-952ac6af647c"
+    // ),
+    // __APP_FRONT__: JSON.stringify("pizza-demo"),
 
     // AI-Workfroce Production settings
-    // __HOME_BASE__: JSON.stringify("https://ai-worker.intelligage.workers.dev"),
-    // __PAP_ID__: undefined,
-    // __APP_FRONT__: JSON.stringify("power-user"),
+    __HOME_BASE__: JSON.stringify("https://ai-worker.intelligage.workers.dev"),
+    __PAP_ID__: undefined,
+    __APP_FRONT__: JSON.stringify("power-user"),
   },
 });