use new URL embedding for 3D avatar

AIPL-labs · Oct 28, 2024 · 84d97ff · 84d97ff
1 parent 60ae733
commit 84d97ff
Show file tree

Hide file tree

Showing 7 changed files with 39 additions and 85 deletions.
diff --git a/bun.lockb b/bun.lockb
diff --git a/package.json b/package.json
@@ -72,7 +72,7 @@
     "@emotion/styled": "^11.13.0",
     "@lunapaint/png-codec": "^0.2.0",
     "@mjtdev/engine": "https://github.com/matthewjosephtaylor/mjtdev-engine#2024.10.25-1350",
-    "ai-worker-common": "https://github.com/AIPL-labs/ai-worker-common#2024.10.25-1357",
+    "ai-worker-common": "https://github.com/AIPL-labs/ai-worker-common#2024.10.28-1208",
     "@mjtdev/avatar-3d": "https://github.com/matthewjosephtaylor/avatar-3d#2024.10.28-0933",
     "@monaco-editor/react": "^4.6.0",
     "@mui/material": "^6.0.1",

diff --git a/src/ui/avatar3d/Avatar3dCharacterEditorContent.tsx b/src/ui/avatar3d/Avatar3dCharacterEditorContent.tsx
@@ -1,48 +1,27 @@
 import { Avatar3dGltf } from "@mjtdev/avatar-3d";
-import { Bytes, Dropzone, type ByteLike } from "@mjtdev/engine";
 import { Flex } from "@radix-ui/themes";
 import type { AppCharacter } from "ai-worker-common";
-import { useEffect, useState } from "react";
+import { FormInputDisplay } from "../form/FormInputDisplay";
 
 export const Avatar3dCharacterEditorContent = ({
   value,
   onChange,
 }: {
   character: AppCharacter;
-  value: ByteLike | undefined;
-  onChange: (value: ByteLike) => void;
+  value: string | undefined;
+  onChange: (value: string) => void;
 }) => {
-  const [state, setState] = useState({
-    path: undefined as string | File | undefined,
-  });
-  useEffect(() => {
-    if (!value) {
-      return;
-    }
-
-    const blob = Bytes.toBlob(value, "model/gltf-binary");
-    const file = new File([blob], "modal.glb");
-
-    setState((s) => ({ ...s, path: file }));
-  }, [value]);
   return (
     <Flex direction={"column"} gap={"2"}>
-      <Dropzone
-        iconSize="4em"
-        iconCode="file_upload"
-        inactiveText={`Add 3D model`}
-        action={async (files: File[]): Promise<void> => {
-          if (!files || files.length < 1) {
-            return;
-          }
-          const ab = await files[0].arrayBuffer();
-          const modelBinary = Bytes.toBlob(ab, "model/gltf-binary");
-          onChange(modelBinary);
-        }}
+      <FormInputDisplay
+        title={"3D Model URL"}
+        style={{ width: "80ch" }}
+        value={value}
+        onChange={onChange}
       />
-      {state.path && (
+      {value && (
         <>
-          <Avatar3dGltf path={state.path} />
+          <Avatar3dGltf path={value} />
         </>
       )}
     </Flex>

diff --git a/src/ui/character/CharacterAvatar.tsx b/src/ui/character/CharacterAvatar.tsx
@@ -1,16 +1,11 @@
 import { Avatar3dGltf } from "@mjtdev/avatar-3d";
-import {
-  Bytes,
-  Colors,
-  Objects,
-  isDefined,
-  type ByteLike,
-} from "@mjtdev/engine";
+import { Colors, Objects, isDefined, type ByteLike } from "@mjtdev/engine";
 import { Badge, Card, Flex, Separator, Text } from "@radix-ui/themes";
 import type { AppCharacter } from "ai-worker-common";
 import { AnimatePresence, motion } from "framer-motion";
 import type { CSSProperties } from "react";
-import { memo, useEffect, useState } from "react";
+import { memo, useState } from "react";
+import { useTtsState } from "../../tts/TtsState";
 import { DEFAULT_CHAR_URL } from "../DEFAULT_CHAR_URL";
 import { stringifyEq } from "../chat/stringifyEq";
 import { AppButtonGroup } from "../common/AppButtonGroup";
@@ -22,7 +17,6 @@ import { idToColor } from "../visual/idToColor";
 import { VideoPlayer } from "./VideoPlayer";
 import type { CharacterAction } from "./characterToActions";
 import { characterToActions } from "./characterToActions";
-import { useTtsState } from "../../tts/TtsState";
 
 export const CharacterAvatar = memo(
   ({
@@ -31,20 +25,19 @@ export const CharacterAvatar = memo(
     nameStyle = {},
     character,
     video,
-    avatar3d,
     showName = true,
     showTags = false,
     showNotes = false,
     showHoverButtons = true,
     showActionButtons = false,
     showContextMenu = true,
+    show3dAvatar = false,
     enableDocumentDrop = true,
     hoverActions,
     buttonActions = [],
     onClick = () => {},
   }: {
     video?: ByteLike;
-    avatar3d?: ByteLike;
     enableDocumentDrop?: boolean;
     showHoverButtons?: boolean;
     hoverActions?: CharacterAction[];
@@ -54,6 +47,7 @@ export const CharacterAvatar = memo(
     showTags?: boolean;
     showNotes?: boolean;
     showContextMenu?: boolean;
+    show3dAvatar?: boolean;
     imageStyle?: CSSProperties;
     nameStyle?: CSSProperties;
     style?: CSSProperties;
@@ -62,19 +56,8 @@ export const CharacterAvatar = memo(
   }) => {
     const [pointerOver, setPointerOver] = useState(false);
     const [videoEnded, setVideoEnded] = useState(false);
-    const [avatar3dFile, setAvatar3dFile] = useState<File | undefined>();
     const { analyserNode } = useTtsState();
 
-    useEffect(() => {
-      if (!avatar3d) {
-        return;
-      }
-
-      const blob = Bytes.toBlob(avatar3d, "model/gltf-binary");
-      const file = new File([blob], "modal.glb");
-      setAvatar3dFile(file);
-    }, [avatar3d]);
-
     const backgroundColor = Colors.from("black").alpha(0.5).toString();
     const nameDisplay = showName ? (
       <Text style={{ ...nameStyle }}>{character?.card?.data?.name}</Text>
@@ -184,10 +167,11 @@ export const CharacterAvatar = memo(
                     exit={{ opacity: 0, transition: { duration: 0.1 } }}
                     key="greeting-image-active-motion"
                   >
-                    {avatar3dFile ? (
+                    {show3dAvatar &&
+                    character.card.data.extensions?.avatar3dUrl ? (
                       <Flex style={{ maxHeight: "40vh", overflow: "auto" }}>
                         <Avatar3dGltf
-                          path={avatar3dFile}
+                          path={character.card.data.extensions.avatar3dUrl}
                           analyserNode={analyserNode}
                           // showPhonemes={true}
                           // showControls={true}

diff --git a/src/ui/character/CharacterEditor.tsx b/src/ui/character/CharacterEditor.tsx
@@ -78,10 +78,6 @@ export const CharacterEditor = ({
     Record<string, ByteLike | undefined>
   >({});
 
-  const [resultAvatar3d, setResultAvatar3d] = useState<ByteLike | undefined>(
-    undefined
-  );
-
   useEffect(() => {
     if (!character.imageDataId) {
       Images.toBlob(DEFAULT_CHAR_URL).then((blob) => {
@@ -91,21 +87,17 @@ export const CharacterEditor = ({
     }
     DatasState.getData(character.imageDataId).then(async (blob) => {
       setResultImage(blob);
-      const { voiceSample, videoPack, avatar3d } =
+      const { voiceSample, videoPack } =
         await AppImages.pngToTavernCardAndVoiceSample(blob, {
-          extraExtractions: ["avatar3d", "videoPack", "voiceSample"],
+          extraExtractions: ["videoPack", "voiceSample"],
         });
-      console.log("avatar3d", avatar3d);
       if (isDefined(voiceSample)) {
         setResultVoiceSample(voiceSample.slice(0));
       }
       if (isDefined(videoPack)) {
         const videos = AppVideos.videoPackToVideoRecords(videoPack);
         setResultVideos(videos);
       }
-      if (isDefined(avatar3d)) {
-        setResultAvatar3d(avatar3d);
-      }
     });
   }, [character.id, character.imageDataId]);
   if (!resultCharacter) {
@@ -289,10 +281,17 @@ export const CharacterEditor = ({
       >
         <Avatar3dCharacterEditorContent
           character={resultCharacter}
+          value={resultCharacter.card.data.extensions?.avatar3dUrl}
           onChange={(value) => {
-            setResultAvatar3d(value);
+            setResultCharacter(
+              produce(resultCharacter, (r) => {
+                r.card.data.extensions = {
+                  ...(r.card.data.extensions ?? {}),
+                  avatar3dUrl: value,
+                };
+              })
+            );
           }}
-          value={resultAvatar3d}
         />
       </Tabs.Content>
     ),
@@ -728,7 +727,6 @@ export const CharacterEditor = ({
                     voiceSample: resultVoiceSample,
                     activeGroupId: resultActiveGroupId,
                     videos: resultVideos,
-                    avatar3d: resultAvatar3d,
                   })
                 }
               >

diff --git a/src/ui/character/updateAppCharacter.ts b/src/ui/character/updateAppCharacter.ts
@@ -3,11 +3,10 @@ import type { AppCharacter, DecomposedAppCharacter } from "ai-worker-common";
 import { AppImages, uniqueId } from "ai-worker-common";
 import { AppEvents } from "../../event/AppEvents";
 import { DataObjectStates } from "../../state/data-object/DataObjectStates";
-import { DatasState } from "../../state/data/DatasState";
+import { Returns } from "../../state/data-object/Returns";
 import { getUserState } from "../../state/user/UserState";
 import { switchActiveGroup } from "../../state/user/switchActiveGroup";
 import { AppMessagesState } from "../../state/ws/AppMessagesState";
-import { Returns } from "../../state/data-object/Returns";
 
 export const updateAppCharacter = async (
   decomposedAppCharacter: DecomposedAppCharacter | undefined
@@ -38,16 +37,10 @@ export const updateAppCharacter = async (
     voiceSample: decomposedAppCharacter.voiceSample,
     activeGroupId: decomposedAppCharacter.activeGroupId,
     videos: decomposedAppCharacter.videos,
-    avatar3d: decomposedAppCharacter.avatar3d,
   });
 
   console.log("updateAppCharacter imageBlob", imageBlob);
 
-  // await DatasState.putBlob({
-  //   blob: imageBlob,
-  //   id: imageDataId,
-  // });
-
   const returnId = Returns.addReturnListener({
     onReturn: async (value) => {
       console.log("updateAppCharacter: return from putting image", value);

diff --git a/src/ui/chat/ChatBox.tsx b/src/ui/chat/ChatBox.tsx
@@ -46,7 +46,7 @@ export const ChatBox = memo(
     const [windowHeight, setWindowHeight] = useState(0);
     const { modes } = AppModes.useAppModesAndParams();
     // const { audioContext: ttsAudioContext, analyserNode } = getTtsState();
-    const {  analyserNode } = getTtsState();
+    const { analyserNode } = getTtsState();
     const { appearance } = useAppState();
 
     const ttsSpeaking = useIsTtsSpeaking();
@@ -80,15 +80,15 @@ export const ChatBox = memo(
     const [greetingVideo, setGreetingVideo] = useState<ByteLike | undefined>(
       undefined
     );
-    const [avatar3d, setAvatar3d] = useState<ByteLike | undefined>(undefined);
     useEffect(() => {
       if (isDefined(aiCharacter?.imageDataId)) {
         DatasState.getData(aiCharacter.imageDataId).then(async (blob) => {
-          const { videoPack, avatar3d } =
-            await AppImages.pngToTavernCardAndVoiceSample(blob, {
-              extraExtractions: ["videoPack", "avatar3d"],
-            });
-          setAvatar3d(avatar3d);
+          const { videoPack } = await AppImages.pngToTavernCardAndVoiceSample(
+            blob,
+            {
+              extraExtractions: ["videoPack"],
+            }
+          );
           if (isDefined(videoPack)) {
             const videos = AppVideos.videoPackToVideoRecords(videoPack);
             setGreetingVideo(videos["greeting"]);
@@ -109,8 +109,8 @@ export const ChatBox = memo(
         hoverActions={["Chat With {char}"]}
         imageStyle={{ maxWidth: "4em" }}
         character={aiCharacter}
+        show3dAvatar={true}
         video={greetingVideo}
-        avatar3d={avatar3d}
       />
     ) : undefined;
     return (