fix(hand tracking): fix event order when using hands and hide hands i…

CONTRIBUTING.md

@@ -12,7 +12,7 @@
 
 ## Run locally
 
-* Run `yarn dev` to be able to to run examples
+* Run `yarn dev` to be able to run examples
 
 ## PR checklist
 
@@ -27,4 +27,4 @@
 ### Test examples locally
 
 Using real device is recommended, otherwise you can use https://github.com/meta-quest/immersive-web-emulator/.
-See [Run locally](#run-locally)
+See [Run locally](#run-locally)

examples/src/demos/Teleport.tsx

@@ -1,5 +1,5 @@
 import { Canvas } from '@react-three/fiber'
-import { Hands, XR, VRButton, TeleportationPlane, Controllers } from '@react-three/xr'
+import { XR, VRButton, TeleportationPlane, Controllers } from '@react-three/xr'
 
 export default function () {
   return (

src/Controllers.tsx

@@ -64,10 +64,16 @@ class ControllerModel extends THREE.Group {
   }
 
   private _onConnected(event: XRControllerEvent) {
+    if (event.data?.hand) {
+      return
+    }
     modelFactory.initializeControllerModel(this.xrControllerModel, event)
   }
 
-  private _onDisconnected(_event: XRControllerEvent) {
+  private _onDisconnected(event: XRControllerEvent) {
+    if (event.data?.hand) {
+      return
+    }
     this.xrControllerModel.disconnect()
   }
 

src/Hands.tsx

@@ -1,6 +1,6 @@
 import * as React from 'react'
 import { Object3DNode, extend, createPortal } from '@react-three/fiber'
-import { OculusHandModel } from 'three-stdlib'
+import { OculusHandModel } from './OculusHandModel'
 import { useXR } from './XR'
 import { useIsomorphicLayoutEffect } from './utils'
 

src/OculusHandModel.ts

@@ -0,0 +1,117 @@
+import { Object3D, Sphere, Box3, Mesh, Texture, Vector3, EventListener, Event } from 'three'
+import { XRHandMeshModel } from './XRHandMeshModel'
+const TOUCH_RADIUS = 0.01
+const POINTING_JOINT = 'index-finger-tip'
+
+export interface XRButton extends Object3D {
+  onPress(): void
+  onClear(): void
+  isPressed(): boolean
+  whilePressed(): void
+}
+
+class OculusHandModel extends Object3D {
+  controller: Object3D
+  motionController: XRHandMeshModel | null
+  envMap: Texture | null
+  mesh: Mesh | null
+  xrInputSource: XRInputSource | null
+
+  leftModelPath?: string
+  rightModelPath?: string
+
+  constructor(controller: Object3D, leftModelPath?: string, rightModelPath?: string) {
+    super()
+
+    this.controller = controller
+    this.motionController = null
+    this.envMap = null
+    this.leftModelPath = leftModelPath
+    this.rightModelPath = rightModelPath
+
+    this.mesh = null
+    this.xrInputSource = null
+
+    controller.addEventListener('connected', this._onConnected)
+    controller.addEventListener('disconnected', this._onDisconnected)
+  }
+
+  private _onConnected: EventListener<Event, 'connected', Object3D<Event>> = (event) => {
+    const xrInputSource = event.data
+
+    if (xrInputSource.hand && !this.motionController) {
+      this.xrInputSource = xrInputSource
+
+      this.motionController = new XRHandMeshModel(
+        this,
+        this.controller,
+        undefined,
+        xrInputSource.handedness,
+        xrInputSource.handedness === 'left' ? this.leftModelPath : this.rightModelPath
+      )
+    }
+  }
+
+  private _onDisconnected: EventListener<Event, 'disconnected', Object3D<Event>> = () => {
+    if (!this.xrInputSource?.hand) {
+      return;
+    }
+    this.motionControllerCleanup()
+  }
+
+  private motionControllerCleanup(): void {
+    this.clear()
+    this.motionController?.dispose()
+    this.motionController = null
+  }
+
+  updateMatrixWorld(force?: boolean): void {
+    super.updateMatrixWorld(force)
+
+    if (this.motionController) {
+      this.motionController.updateMesh()
+    }
+  }
+
+  getPointerPosition(): Vector3 | null {
+    // @ts-ignore XRController needs to extend Group
+    const indexFingerTip = this.controller.joints[POINTING_JOINT]
+    if (indexFingerTip) {
+      return indexFingerTip.position
+    } else {
+      return null
+    }
+  }
+
+  intersectBoxObject(boxObject: Object3D): boolean {
+    const pointerPosition = this.getPointerPosition()
+    if (pointerPosition) {
+      const indexSphere = new Sphere(pointerPosition, TOUCH_RADIUS)
+      const box = new Box3().setFromObject(boxObject)
+      return indexSphere.intersectsBox(box)
+    } else {
+      return false
+    }
+  }
+
+  checkButton(button: XRButton): void {
+    if (this.intersectBoxObject(button)) {
+      button.onPress()
+    } else {
+      button.onClear()
+    }
+
+    if (button.isPressed()) {
+      button.whilePressed()
+    }
+  }
+
+  dispose(): void {
+    this.motionControllerCleanup()
+
+    this.controller.removeEventListener('connected', this._onConnected)
+    this.controller.removeEventListener('disconnected', this._onDisconnected)
+  }
+}
+
+export { OculusHandModel }

src/XRHandMeshModel.ts

@@ -0,0 +1,113 @@
+import { Object3D } from 'three'
+import { GLTFLoader } from 'three-stdlib'
+
+const DEFAULT_HAND_PROFILE_PATH = 'https://cdn.jsdelivr.net/npm/@webxr-input-profiles/[email protected]/dist/profiles/generic-hand/'
+
+class XRHandMeshModel {
+  controller: Object3D
+  handModel: Object3D
+  bones: Object3D[]
+  scene?: Object3D
+
+  constructor(
+    handModel: Object3D,
+    controller: Object3D,
+    path: string = DEFAULT_HAND_PROFILE_PATH,
+    handedness: string,
+    customModelPath?: string
+  ) {
+    this.controller = controller
+    this.handModel = handModel
+
+    this.bones = []
+
+    const loader = new GLTFLoader()
+    if (!customModelPath) loader.setPath(path)
+    loader.load(customModelPath ?? `${handedness}.glb`, (gltf: { scene: Object3D }) => {
+      const object = gltf.scene.children[0]
+      this.handModel.add(object)
+      this.scene = object
+
+      const mesh = object.getObjectByProperty('type', 'SkinnedMesh')!
+      mesh.frustumCulled = false
+      mesh.castShadow = true
+      mesh.receiveShadow = true
+
+      const joints = [
+        'wrist',
+        'thumb-metacarpal',
+        'thumb-phalanx-proximal',
+        'thumb-phalanx-distal',
+        'thumb-tip',
+        'index-finger-metacarpal',
+        'index-finger-phalanx-proximal',
+        'index-finger-phalanx-intermediate',
+        'index-finger-phalanx-distal',
+        'index-finger-tip',
+        'middle-finger-metacarpal',
+        'middle-finger-phalanx-proximal',
+        'middle-finger-phalanx-intermediate',
+        'middle-finger-phalanx-distal',
+        'middle-finger-tip',
+        'ring-finger-metacarpal',
+        'ring-finger-phalanx-proximal',
+        'ring-finger-phalanx-intermediate',
+        'ring-finger-phalanx-distal',
+        'ring-finger-tip',
+        'pinky-finger-metacarpal',
+        'pinky-finger-phalanx-proximal',
+        'pinky-finger-phalanx-intermediate',
+        'pinky-finger-phalanx-distal',
+        'pinky-finger-tip'
+      ]
+
+      joints.forEach((jointName) => {
+        const bone = object.getObjectByName(jointName) as any
+
+        if (bone !== undefined) {
+          bone.jointName = jointName
+        } else {
+          console.warn(`Couldn't find ${jointName} in ${handedness} hand mesh`)
+        }
+
+        this.bones.push(bone)
+      })
+    })
+  }
+
+  updateMesh(): void {
+    // XR Joints
+    const XRJoints = (this.controller as any).joints
+    let allInvisible = true
+
+    for (let i = 0; i < this.bones.length; i++) {
+      const bone = this.bones[i]
+
+      if (bone) {
+        const XRJoint = XRJoints[(bone as any).jointName]
+
+        if (XRJoint.visible) {
+          const position = XRJoint.position
+          bone.position.copy(position)
+          bone.quaternion.copy(XRJoint.quaternion)
+          allInvisible = false
+        }
+      }
+    }
+
+    // Hide hand mesh if all joints are invisible in case hand loses tracking
+    if (allInvisible && this.scene) {
+      this.scene.visible = false
+    } else if (this.scene) {
+      this.scene.visible = true
+    }
+  }
+
+  dispose(): void {
+    if (this.scene) {
+      this.handModel.remove(this.scene)
+    }
+  }
+}
+
+export { XRHandMeshModel }

yarn.lock

@@ -10,7 +10,6 @@
     "@jridgewell/gen-mapping" "^0.1.0"
     "@jridgewell/trace-mapping" "^0.3.9"
 
-
 "@babel/code-frame@^7.18.6":
   version "7.18.6"
   resolved "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.18.6.tgz"
@@ -755,6 +754,11 @@
   resolved "https://registry.npmjs.org/@types/chai/-/chai-4.3.4.tgz"
   integrity sha512-KnRanxnpfpjUTqTCXslZSEdLfXExwgNxYPdiO2WGUj8+HDjFi8R3k5RVKPeSCzLjCcshCAtVO2QBbVuAV4kTnw==
 
+"@types/draco3d@^1.4.0":
+  version "1.4.2"
+  resolved "https://registry.yarnpkg.com/@types/draco3d/-/draco3d-1.4.2.tgz#7faccb809db2a5e19b9efb97c5f2eb9d64d527ea"
+  integrity sha512-goh23EGr6CLV6aKPwN1p8kBD/7tT5V/bLpToSbarKrwVejqNrspVrv8DhliteYkkhZYrlq/fwKZRRUzH4XN88w==
+
 "@types/json-schema@^7.0.9":
   version "7.0.11"
   resolved "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.11.tgz"
@@ -827,6 +831,11 @@
   resolved "https://registry.npmjs.org/@types/webxr/-/webxr-0.4.0.tgz"
   integrity sha512-LQvrACV3Pj17GpkwHwXuTd733gfY+D7b9mKdrTmLdO7vo7P/o6209Qqtk63y/FCv/lspdmi0pWz6Qe/ull9kQg==
 
+"@types/webxr@^0.5.2":
+  version "0.5.2"
+  resolved "https://registry.yarnpkg.com/@types/webxr/-/webxr-0.5.2.tgz#5d9627b0ffe223aa3b166de7112ac8a9460dc54f"
+  integrity sha512-szL74BnIcok9m7QwYtVmQ+EdIKwbjPANudfuvDrAF8Cljg9MKUlIoc1w5tjj9PMpeSH3U1Xnx//czQybJ0EfSw==
+
 "@typescript-eslint/eslint-plugin@^5.11.0":
   version "5.28.0"
   resolved "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.28.0.tgz"
@@ -2337,11 +2346,6 @@ jsonc-parser@^3.2.0:
     array-includes "^3.1.4"
     object.assign "^4.1.2"
 
-ktx-parse@^0.2.1:
-  version "0.2.2"
-  resolved "https://registry.npmjs.org/ktx-parse/-/ktx-parse-0.2.2.tgz"
-  integrity sha512-cFBc1jnGG2WlUf52NbDUXK2obJ+Mo9WUkBRvr6tP6CKxRMvZwDDFNV3JAS4cewETp5KyexByfWm9sm+O8AffiQ==
-
 ktx-parse@^0.4.5:
   version "0.4.5"
   resolved "https://registry.npmjs.org/ktx-parse/-/ktx-parse-0.4.5.tgz"
@@ -3187,16 +3191,17 @@ three-mesh-bvh@^0.5.10:
   integrity sha512-IMNHrAnsLCIxcFmAGkA4Wibw1QEpFQlkR72XUxZFOatNSpfMRUhJXQwQ5jPxbrX0W+OR838t/IR3laMOvQnT/g==
 
 three-stdlib@^2.10.2:
-  version "2.12.1"
-  resolved "https://registry.npmjs.org/three-stdlib/-/three-stdlib-2.12.1.tgz"
-  integrity sha512-G3SSsCOBiWa0sjjPt+K28ikQ84Plm/ZVUozMfWagK59kZqBWcaPVXpOThkAgvdBpm2zCWLW3edAoW/4XIbljVQ==
+  version "2.23.10"
+  resolved "https://registry.yarnpkg.com/three-stdlib/-/three-stdlib-2.23.10.tgz#94907a558a00da327bd74308c92078fea72f77fc"
+  integrity sha512-y0DlxaN5HZXI9hKjEtqO2xlCEt7XyDCOMvD2M3JJFBmYjwbU+PbJ1n3Z+7Hr/6BeVGE6KZYcqPMnfKrTK5WTJg==
   dependencies:
-    "@babel/runtime" "^7.16.7"
-    "@webgpu/glslang" "^0.0.15"
+    "@types/draco3d" "^1.4.0"
+    "@types/offscreencanvas" "^2019.6.4"
+    "@types/webxr" "^0.5.2"
     chevrotain "^10.1.2"
     draco3d "^1.4.1"
     fflate "^0.6.9"
-    ktx-parse "^0.2.1"
+    ktx-parse "^0.4.5"
     mmd-parser "^1.0.4"
     opentype.js "^1.3.3"
     potpack "^1.0.1"