Add basic support for loading media with bitecs behind feature flag.

babel.config.js

@@ -4,7 +4,6 @@ module.exports = {
     [
       "@babel/env",
       {
-        exclude: ["transform-regenerator"],
         // targets are defined in .browserslistrc
         useBuiltIns: "entry",
         // This should be kept up to date with thee version in package.json

src/app.ts

@@ -23,6 +23,7 @@ import {
 } from "three";
 import { AudioSettings, SourceType } from "./components/audio-params";
 import { DialogAdapter } from "./naf-dialog-adapter";
+import { waitForPreloads } from "./utils/preload";
 
 declare global {
   interface Window {
@@ -33,7 +34,7 @@ declare global {
   const APP: App;
 }
 
-interface HubsWorld extends IWorld {
+export interface HubsWorld extends IWorld {
   scene: Scene;
   nameToComponent: {
     object3d: typeof Object3DTag;
@@ -69,17 +70,17 @@ export class App {
   store = new Store();
   mediaSearchStore = new MediaSearchStore();
 
-  audios = new Map<AElement, PositionalAudio | Audio>();
-  sourceType = new Map<AElement, SourceType>();
-  audioOverrides = new Map<AElement, AudioSettings>();
-  zoneOverrides = new Map<AElement, AudioSettings>();
+  audios = new Map<AElement | number, PositionalAudio | Audio>();
+  sourceType = new Map<AElement | number, SourceType>();
+  audioOverrides = new Map<AElement | number, AudioSettings>();
+  zoneOverrides = new Map<AElement | number, AudioSettings>();
+  gainMultipliers = new Map<AElement | number, number>();
+  supplementaryAttenuation = new Map<AElement | number, number>();
+  clippingState = new Set<AElement | number>();
+  mutedState = new Set<AElement | number>();
+  isAudioPaused = new Set<AElement | number>();
   audioDebugPanelOverrides = new Map<SourceType, AudioSettings>();
   sceneAudioDefaults = new Map<SourceType, AudioSettings>();
-  gainMultipliers = new Map<AElement, number>();
-  supplementaryAttenuation = new Map<AElement, number>();
-  clippingState = new Set<AElement>();
-  mutedState = new Set<AElement>();
-  isAudioPaused = new Set<AElement>();
 
   world: HubsWorld = createWorld();
 
@@ -218,8 +219,10 @@ export class App {
 
     // This gets called after all system and component init functions
     sceneEl.addEventListener("loaded", () => {
-      renderer.setAnimationLoop(mainTick);
-      sceneEl.renderStarted = true;
+      waitForPreloads().then(() => {
+        renderer.setAnimationLoop(mainTick);
+        sceneEl.renderStarted = true;
+      });
     });
 
     return {

src/bit-components.js

@@ -67,6 +67,11 @@ export const HeldHandLeft = defineComponent();
 export const HeldRemoteRight = defineComponent();
 export const HeldRemoteLeft = defineComponent();
 export const Held = defineComponent();
+export const Constraint = defineComponent();
+export const ConstraintHandRight = defineComponent();
+export const ConstraintHandLeft = defineComponent();
+export const ConstraintRemoteRight = defineComponent();
+export const ConstraintRemoteLeft = defineComponent();
 export const OffersRemoteConstraint = defineComponent();
 export const HandCollisionTarget = defineComponent();
 export const OffersHandConstraint = defineComponent();
@@ -120,3 +125,38 @@ export const CameraTool = defineComponent({
   sndToggleRef: Types.eid
 });
 export const MyCameraTool = defineComponent();
+export const MediaLoader = defineComponent({
+  src: Types.ui32,
+  flags: Types.ui8
+});
+MediaLoader.src[$isStringType] = true;
+
+export const TextureCacheKey = defineComponent({
+  src: Types.ui32,
+  version: Types.ui8
+});
+TextureCacheKey.src[$isStringType] = true;
+
+export const MediaVideo = defineComponent({
+  autoPlay: Types.ui8
+});
+
+export const AnimationMixer = defineComponent();
+export const NetworkedVideo = defineComponent({
+  time: Types.f32,
+  flags: Types.ui8
+});
+
+export const VideoMenuItem = defineComponent();
+export const VideoMenu = defineComponent({
+  videoRef: Types.eid,
+  timeLabelRef: Types.eid,
+  trackRef: Types.eid,
+  headRef: Types.eid,
+  playIndicatorRef: Types.eid,
+  pauseIndicatorRef: Types.eid
+});
+
+export const AudioEmitter = defineComponent();
+export const AudioSettingsChanged = defineComponent();
+export const Deletable = defineComponent();

src/bit-systems/camera-tool.js

@@ -208,7 +208,7 @@ function updateUI(world, camera) {
     CameraTool.state[camera] === CAMERA_STATE.RECORDING_VIDEO;
 
   const inVR = AFRAME.scenes[0].is("vr-mode");
-  const showViewfinder = !inVR || (hasComponent(world, Held, camera) || !isIdle);
+  const showViewfinder = !inVR || hasComponent(world, Held, camera) || !isIdle;
   screenObj.visible = showViewfinder;
   selfieScreenObj.visible = showViewfinder;
 
@@ -249,7 +249,8 @@ function updateUI(world, camera) {
 
   // TODO HACK hidden objects are still not having their matricies updated correctly
   // Seems like a regression of #5421
-  snapMenuObj.matrixNeedsUpdate = true;
+  // updateMatrices should be checking forceWorldUpdate instead of parent.childrenNeedMatrixWorldUpdate
+  snapMenuObj.childrenNeedMatrixWorldUpdate = true;
 }
 
 let snapPixels;
@@ -289,7 +290,7 @@ const cameraToolEnterQuery = enterQuery(cameraToolQuery);
 const cameraToolExitQuery = exitQuery(cameraToolQuery);
 
 export function cameraToolSystem(world) {
-  cameraToolEnterQuery(world).forEach(function(eid) {
+  cameraToolEnterQuery(world).forEach(function (eid) {
     const renderTarget = new THREE.WebGLRenderTarget(RENDER_WIDTH, RENDER_HEIGHT, {
       format: THREE.RGBAFormat,
       minFilter: THREE.LinearFilter,
@@ -317,7 +318,7 @@ export function cameraToolSystem(world) {
     renderTargets.set(eid, renderTarget);
   });
 
-  cameraToolExitQuery(world).forEach(function(eid) {
+  cameraToolExitQuery(world).forEach(function (eid) {
     const renderTarget = renderTargets.get(eid);
     renderTarget.dispose();
     renderTargets.delete(eid);

src/bit-systems/delete-entity-system.ts

@@ -0,0 +1,54 @@
+import { defineQuery, exitQuery, hasComponent, removeEntity } from "bitecs";
+import { Vector3 } from "three";
+import { HubsWorld } from "../app";
+import { Deletable, HoveredRemoteLeft, HoveredRemoteRight } from "../bit-components";
+import { paths } from "../systems/userinput/paths";
+import { animate } from "../utils/animate";
+import { findAncestorEntity } from "../utils/bit-utils";
+import { coroutine } from "../utils/coroutine";
+import { easeOutQuadratic } from "../utils/easing";
+
+// TODO Move to coroutine.ts when it exists
+// TODO Figure out the appropriate type and use it everywhere
+export type Coroutine = Generator<Promise<void>, void, unknown>;
+
+const END_SCALE = new Vector3().setScalar(0.001);
+function* animateThenRemoveEntity(world: HubsWorld, eid: number): Coroutine {
+  const obj = world.eid2obj.get(eid)!;
+  yield* animate({
+    properties: [[obj.scale.clone(), END_SCALE]],
+    durationMS: 400,
+    easing: easeOutQuadratic,
+    fn: ([scale]: [Vector3]) => {
+      obj.scale.copy(scale);
+      obj.matrixNeedsUpdate = true;
+    }
+  });
+  removeEntity(world, eid);
+}
+
+const deletableQuery = defineQuery([Deletable]);
+const deletableExitQuery = exitQuery(deletableQuery);
+const hoveredRightQuery = defineQuery([HoveredRemoteRight]);
+const hoveredLeftQuery = defineQuery([HoveredRemoteLeft]);
+const coroutines = new Map();
+
+function deleteTheDeletableAncestor(world: HubsWorld, eid: number) {
+  const ancestor = findAncestorEntity(world, eid, (e: number) => hasComponent(world, Deletable, e));
+  if (ancestor && !coroutines.has(ancestor)) {
+    coroutines.set(ancestor, coroutine(animateThenRemoveEntity(world, ancestor)));
+  }
+}
+
+export function deleteEntitySystem(world: HubsWorld, userinput: any) {
+  deletableExitQuery(world).forEach(function (eid) {
+    coroutines.delete(eid);
+  });
+  if (userinput.get(paths.actions.cursor.right.deleteEntity)) {
+    hoveredRightQuery(world).forEach(eid => deleteTheDeletableAncestor(world, eid));
+  }
+  if (userinput.get(paths.actions.cursor.left.deleteEntity)) {
+    hoveredLeftQuery(world).forEach(eid => deleteTheDeletableAncestor(world, eid));
+  }
+  coroutines.forEach(c => c());
+}

src/bit-systems/media-loading.js

@@ -0,0 +1,166 @@
+import { LoadingObject } from "../prefabs/loading-object";
+import { ErrorObject } from "../prefabs/error-object";
+import { easeOutQuadratic } from "../utils/easing";
+import { loadImage } from "../utils/load-image";
+import { loadVideo } from "../utils/load-video";
+import { loadModel } from "../utils/load-model";
+import { MediaType, resolveMediaInfo } from "../utils/media-utils";
+import { defineQuery, enterQuery, exitQuery, hasComponent, removeComponent, removeEntity } from "bitecs";
+import { MediaLoader, Networked } from "../bit-components";
+import { crTimeout, crClearTimeout, cancelable, coroutine, makeCancelable } from "../utils/coroutine";
+import { takeOwnership } from "../systems/netcode";
+import { renderAsEntity } from "../utils/jsx-entity";
+import { animate } from "../utils/animate";
+
+const loaderForMediaType = {
+  [MediaType.IMAGE]: (world, { accessibleUrl, contentType }) => loadImage(world, accessibleUrl, contentType),
+  [MediaType.VIDEO]: (world, { accessibleUrl }) => loadVideo(world, accessibleUrl),
+  [MediaType.MODEL]: (world, { accessibleUrl }) => loadModel(world, accessibleUrl)
+};
+
+export const MEDIA_LOADER_FLAGS = {
+  RECENTER: 1 << 0,
+  RESIZE: 1 << 1
+};
+
+function assignNetworkIds(world, mediaEid, mediaLoaderEid) {
+  const rootNid = APP.getString(Networked.id[mediaLoaderEid]);
+  let i = 0;
+  world.eid2obj.get(mediaEid).traverse(function (obj) {
+    if (obj.eid && hasComponent(world, Networked, obj.eid)) {
+      const eid = obj.eid;
+      Networked.id[eid] = APP.getSid(`${rootNid}.${i}`);
+      APP.world.nid2eid.set(Networked.id[eid], eid);
+      Networked.creator[eid] = Networked.creator[mediaLoaderEid];
+      Networked.owner[eid] = Networked.owner[mediaLoaderEid];
+      if (APP.getSid(NAF.clientId) === Networked.owner[mediaLoaderEid]) takeOwnership(world, eid);
+      i += 1;
+    }
+  });
+}
+
+function resizeAndRecenter(world, media, eid) {
+  const resize = MediaLoader.flags[eid] & MEDIA_LOADER_FLAGS.RESIZE;
+  const recenter = MediaLoader.flags[eid] & MEDIA_LOADER_FLAGS.RECENTER;
+  if (!resize && !recenter) return;
+
+  const mediaObj = world.eid2obj.get(media);
+  const box = new THREE.Box3();
+  box.setFromObject(mediaObj);
+
+  let scale = 1;
+  if (resize) {
+    const size = new THREE.Vector3();
+    box.getSize(size);
+    // Might want to consider avatar scale someday
+    scale = 0.5 / Math.max(size.x, size.y, size.z);
+    mediaObj.scale.setScalar(scale);
+    mediaObj.matrixNeedsUpdate = true;
+  }
+
+  if (recenter) {
+    const center = new THREE.Vector3();
+    box.getCenter(center);
+    mediaObj.position.copy(center).multiplyScalar(-1 * scale);
+    mediaObj.matrixNeedsUpdate = true;
+  }
+}
+
+function* animateScale(world, media) {
+  const mediaObj = world.eid2obj.get(media);
+
+  const onAnimate = ([position, scale]) => {
+    mediaObj.position.copy(position);
+    mediaObj.scale.copy(scale);
+    mediaObj.matrixNeedsUpdate = true;
+  };
+
+  const startScale = new THREE.Vector3().setScalar(0.0001);
+  const endScale = new THREE.Vector3().setScalar(mediaObj.scale.x);
+
+  const startPosition = new THREE.Vector3().copy(mediaObj.position).multiplyScalar(startScale.x);
+  const endPosition = new THREE.Vector3().copy(mediaObj.position);
+
+  // Set the initial position and yield one frame
+  // because the first frame that we render a new object is slow
+  // TODO: We could move uploading textures to the GPU to the loader,
+  //       so that we don't hitch here
+  onAnimate([startPosition, startScale]);
+  yield Promise.resolve();
+
+  yield* animate({
+    properties: [
+      [startPosition, endPosition],
+      [startScale, endScale]
+    ],
+    durationMS: 400,
+    easing: easeOutQuadratic,
+    fn: onAnimate
+  });
+}
+
+function add(world, child, parent) {
+  const parentObj = world.eid2obj.get(parent);
+  const childObj = world.eid2obj.get(child);
+  parentObj.add(childObj);
+
+  // TODO: Fix this in THREE.Object3D.add
+  childObj.matrixWorldNeedsUpdate = true;
+}
+
+function* loadMedia(world, eid) {
+  let loadingObjEid = 0;
+  const addLoadingObjectTimeout = crTimeout(() => {
+    loadingObjEid = renderAsEntity(world, LoadingObject());
+    add(world, loadingObjEid, eid);
+  }, 400);
+  yield makeCancelable(() => loadingObjEid && removeEntity(world, loadingObjEid));
+  const src = APP.getString(MediaLoader.src[eid]);
+  let media;
+  try {
+    const urlData = yield resolveMediaInfo(src);
+    media = yield* loaderForMediaType[urlData.mediaType](world, urlData);
+  } catch (e) {
+    console.error(e);
+    media = renderAsEntity(world, ErrorObject());
+  }
+  crClearTimeout(addLoadingObjectTimeout);
+  loadingObjEid && removeEntity(world, loadingObjEid);
+  return media;
+}
+
+function* loadAndAnimateMedia(world, eid, signal) {
+  const { value: media, canceled } = yield* cancelable(loadMedia(world, eid), signal);
+  if (!canceled) {
+    assignNetworkIds(world, media, eid);
+    resizeAndRecenter(world, media, eid);
+    add(world, media, eid);
+    yield* animateScale(world, media);
+    removeComponent(world, MediaLoader, eid);
+  }
+}
+
+const jobs = new Set();
+const abortControllers = new Map();
+const mediaLoaderQuery = defineQuery([MediaLoader]);
+const mediaLoaderEnterQuery = enterQuery(mediaLoaderQuery);
+const mediaLoaderExitQuery = exitQuery(mediaLoaderQuery);
+export function mediaLoadingSystem(world) {
+  mediaLoaderEnterQuery(world).forEach(function (eid) {
+    const ac = new AbortController();
+    abortControllers.set(eid, ac);
+    jobs.add(coroutine(loadAndAnimateMedia(world, eid, ac.signal)));
+  });
+
+  mediaLoaderExitQuery(world).forEach(function (eid) {
+    const ac = abortControllers.get(eid);
+    ac.abort();
+    abortControllers.delete(eid);
+  });
+
+  jobs.forEach(c => {
+    if (c().done) {
+      jobs.delete(c);
+    }
+  });
+}