From 05bd4893d9a4933e71fc51d87871b8686f805572 Mon Sep 17 00:00:00 2001 From: Artem Ibatullin Date: Wed, 5 Jul 2023 15:58:45 +0300 Subject: [PATCH 1/2] fix(hand tracking): fix event order when using hands and hide hands if tracking is lost --- CONTRIBUTING.md | 4 +- examples/src/demos/Teleport.tsx | 2 +- src/Controllers.tsx | 8 ++- src/Hands.tsx | 2 +- src/OculusHandModel.ts | 112 ++++++++++++++++++++++++++++++++ src/XRHandMeshModel.ts | 112 ++++++++++++++++++++++++++++++++ yarn.lock | 29 +++++---- 7 files changed, 252 insertions(+), 17 deletions(-) create mode 100644 src/OculusHandModel.ts create mode 100644 src/XRHandMeshModel.ts diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8a817320..1deb6037 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,7 +12,7 @@ ## Run locally -* Run `yarn dev` to be able to to run examples +* Run `yarn dev` to be able to run examples ## PR checklist @@ -27,4 +27,4 @@ ### Test examples locally Using real device is recommended, otherwise you can use https://github.com/meta-quest/immersive-web-emulator/. -See [Run locally](#run-locally) \ No newline at end of file +See [Run locally](#run-locally) diff --git a/examples/src/demos/Teleport.tsx b/examples/src/demos/Teleport.tsx index 163817fa..95da6c76 100644 --- a/examples/src/demos/Teleport.tsx +++ b/examples/src/demos/Teleport.tsx @@ -1,5 +1,5 @@ import { Canvas } from '@react-three/fiber' -import { Hands, XR, VRButton, TeleportationPlane, Controllers } from '@react-three/xr' +import { XR, VRButton, TeleportationPlane, Controllers } from '@react-three/xr' export default function () { return ( diff --git a/src/Controllers.tsx b/src/Controllers.tsx index f323dbea..dbea902b 100644 --- a/src/Controllers.tsx +++ b/src/Controllers.tsx @@ -64,10 +64,16 @@ class ControllerModel extends THREE.Group { } private _onConnected(event: XRControllerEvent) { + if (event.data?.hand) { + return + } modelFactory.initializeControllerModel(this.xrControllerModel, event) } - private _onDisconnected(_event: XRControllerEvent) { + private _onDisconnected(event: XRControllerEvent) { + if (event.data?.hand) { + return + } this.xrControllerModel.disconnect() } diff --git a/src/Hands.tsx b/src/Hands.tsx index 0e5468e0..d1de2731 100644 --- a/src/Hands.tsx +++ b/src/Hands.tsx @@ -1,6 +1,6 @@ import * as React from 'react' import { Object3DNode, extend, createPortal } from '@react-three/fiber' -import { OculusHandModel } from 'three-stdlib' +import { OculusHandModel } from './OculusHandModel' import { useXR } from './XR' import { useIsomorphicLayoutEffect } from './utils' diff --git a/src/OculusHandModel.ts b/src/OculusHandModel.ts new file mode 100644 index 00000000..d6d501ef --- /dev/null +++ b/src/OculusHandModel.ts @@ -0,0 +1,112 @@ +import { Object3D, Sphere, Box3, Mesh, Texture, Vector3, EventListener, Event } from 'three' +import { XRHandMeshModel } from './XRHandMeshModel' +const TOUCH_RADIUS = 0.01 +const POINTING_JOINT = 'index-finger-tip' + +export interface XRButton extends Object3D { + onPress(): void + onClear(): void + isPressed(): boolean + whilePressed(): void +} + +class OculusHandModel extends Object3D { + controller: Object3D + motionController: XRHandMeshModel | null + envMap: Texture | null + mesh: Mesh | null + xrInputSource: XRInputSource | null + + leftModelPath?: string + rightModelPath?: string + + constructor(controller: Object3D, leftModelPath?: string, rightModelPath?: string) { + super() + + this.controller = controller + this.motionController = null + this.envMap = null + this.leftModelPath = leftModelPath + this.rightModelPath = rightModelPath + + this.mesh = null + this.xrInputSource = null + + controller.addEventListener('connected', this._onConnected) + controller.addEventListener('disconnected', this._onDisconnected) + } + + private _onConnected: EventListener> = (event) => { + const xrInputSource = event.data + + if (xrInputSource.hand && !this.motionController) { + this.xrInputSource = xrInputSource + + this.motionController = new XRHandMeshModel( + this, + this.controller, + undefined, + xrInputSource.handedness, + xrInputSource.handedness === 'left' ? this.leftModelPath : this.rightModelPath + ) + } + } + + private _onDisconnected: EventListener> = () => { + this.clear() + this.motionController?.dispose() + this.motionController = null + } + + updateMatrixWorld(force?: boolean): void { + super.updateMatrixWorld(force) + + if (this.motionController) { + this.motionController.updateMesh() + } + } + + getPointerPosition(): Vector3 | null { + // @ts-ignore XRController needs to extend Group + const indexFingerTip = this.controller.joints[POINTING_JOINT] + if (indexFingerTip) { + return indexFingerTip.position + } else { + return null + } + } + + intersectBoxObject(boxObject: Object3D): boolean { + const pointerPosition = this.getPointerPosition() + if (pointerPosition) { + const indexSphere = new Sphere(pointerPosition, TOUCH_RADIUS) + const box = new Box3().setFromObject(boxObject) + return indexSphere.intersectsBox(box) + } else { + return false + } + } + + checkButton(button: XRButton): void { + if (this.intersectBoxObject(button)) { + button.onPress() + } else { + button.onClear() + } + + if (button.isPressed()) { + button.whilePressed() + } + } + + dispose(): void { + this.clear() + this.motionController?.dispose() + this.motionController = null + + this.controller.removeEventListener('connected', this._onConnected) + this.controller.removeEventListener('disconnected', this._onDisconnected) + } +} + +export { OculusHandModel } diff --git a/src/XRHandMeshModel.ts b/src/XRHandMeshModel.ts new file mode 100644 index 00000000..6ffb2ee4 --- /dev/null +++ b/src/XRHandMeshModel.ts @@ -0,0 +1,112 @@ +import { Object3D } from 'three' +import { GLTFLoader } from 'three-stdlib' + +const DEFAULT_HAND_PROFILE_PATH = 'https://cdn.jsdelivr.net/npm/@webxr-input-profiles/assets@1.0/dist/profiles/generic-hand/' + +class XRHandMeshModel { + controller: Object3D + handModel: Object3D + bones: Object3D[] + scene?: Object3D + + constructor( + handModel: Object3D, + controller: Object3D, + path: string = DEFAULT_HAND_PROFILE_PATH, + handedness: string, + customModelPath?: string + ) { + this.controller = controller + this.handModel = handModel + + this.bones = [] + + const loader = new GLTFLoader() + if (!customModelPath) loader.setPath(path) + loader.load(customModelPath ?? `${handedness}.glb`, (gltf: { scene: Object3D }) => { + const object = gltf.scene.children[0] + this.handModel.add(object) + this.scene = object + + const mesh = object.getObjectByProperty('type', 'SkinnedMesh')! + mesh.frustumCulled = false + mesh.castShadow = true + mesh.receiveShadow = true + + const joints = [ + 'wrist', + 'thumb-metacarpal', + 'thumb-phalanx-proximal', + 'thumb-phalanx-distal', + 'thumb-tip', + 'index-finger-metacarpal', + 'index-finger-phalanx-proximal', + 'index-finger-phalanx-intermediate', + 'index-finger-phalanx-distal', + 'index-finger-tip', + 'middle-finger-metacarpal', + 'middle-finger-phalanx-proximal', + 'middle-finger-phalanx-intermediate', + 'middle-finger-phalanx-distal', + 'middle-finger-tip', + 'ring-finger-metacarpal', + 'ring-finger-phalanx-proximal', + 'ring-finger-phalanx-intermediate', + 'ring-finger-phalanx-distal', + 'ring-finger-tip', + 'pinky-finger-metacarpal', + 'pinky-finger-phalanx-proximal', + 'pinky-finger-phalanx-intermediate', + 'pinky-finger-phalanx-distal', + 'pinky-finger-tip' + ] + + joints.forEach((jointName) => { + const bone = object.getObjectByName(jointName) as any + + if (bone !== undefined) { + bone.jointName = jointName + } else { + console.warn(`Couldn't find ${jointName} in ${handedness} hand mesh`) + } + + this.bones.push(bone) + }) + }) + } + + updateMesh(): void { + // XR Joints + const XRJoints = (this.controller as any).joints + let allInvisible = true + + for (let i = 0; i < this.bones.length; i++) { + const bone = this.bones[i] + + if (bone) { + const XRJoint = XRJoints[(bone as any).jointName] + + if (XRJoint.visible) { + const position = XRJoint.position + bone.position.copy(position) + bone.quaternion.copy(XRJoint.quaternion) + allInvisible = false + } + } + } + + if (allInvisible && this.scene) { + this.scene.visible = false + } else if (this.scene) { + this.scene.visible = true + } + } + + dispose(): void { + if (this.scene) { + this.handModel.remove(this.scene) + } + } +} + +export { XRHandMeshModel } diff --git a/yarn.lock b/yarn.lock index 0da614ed..a454c619 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10,7 +10,6 @@ "@jridgewell/gen-mapping" "^0.1.0" "@jridgewell/trace-mapping" "^0.3.9" - "@babel/code-frame@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.18.6.tgz" @@ -755,6 +754,11 @@ resolved "https://registry.npmjs.org/@types/chai/-/chai-4.3.4.tgz" integrity sha512-KnRanxnpfpjUTqTCXslZSEdLfXExwgNxYPdiO2WGUj8+HDjFi8R3k5RVKPeSCzLjCcshCAtVO2QBbVuAV4kTnw== +"@types/draco3d@^1.4.0": + version "1.4.2" + resolved "https://registry.yarnpkg.com/@types/draco3d/-/draco3d-1.4.2.tgz#7faccb809db2a5e19b9efb97c5f2eb9d64d527ea" + integrity sha512-goh23EGr6CLV6aKPwN1p8kBD/7tT5V/bLpToSbarKrwVejqNrspVrv8DhliteYkkhZYrlq/fwKZRRUzH4XN88w== + "@types/json-schema@^7.0.9": version "7.0.11" resolved "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.11.tgz" @@ -827,6 +831,11 @@ resolved "https://registry.npmjs.org/@types/webxr/-/webxr-0.4.0.tgz" integrity sha512-LQvrACV3Pj17GpkwHwXuTd733gfY+D7b9mKdrTmLdO7vo7P/o6209Qqtk63y/FCv/lspdmi0pWz6Qe/ull9kQg== +"@types/webxr@^0.5.2": + version "0.5.2" + resolved "https://registry.yarnpkg.com/@types/webxr/-/webxr-0.5.2.tgz#5d9627b0ffe223aa3b166de7112ac8a9460dc54f" + integrity sha512-szL74BnIcok9m7QwYtVmQ+EdIKwbjPANudfuvDrAF8Cljg9MKUlIoc1w5tjj9PMpeSH3U1Xnx//czQybJ0EfSw== + "@typescript-eslint/eslint-plugin@^5.11.0": version "5.28.0" resolved "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.28.0.tgz" @@ -2337,11 +2346,6 @@ jsonc-parser@^3.2.0: array-includes "^3.1.4" object.assign "^4.1.2" -ktx-parse@^0.2.1: - version "0.2.2" - resolved "https://registry.npmjs.org/ktx-parse/-/ktx-parse-0.2.2.tgz" - integrity sha512-cFBc1jnGG2WlUf52NbDUXK2obJ+Mo9WUkBRvr6tP6CKxRMvZwDDFNV3JAS4cewETp5KyexByfWm9sm+O8AffiQ== - ktx-parse@^0.4.5: version "0.4.5" resolved "https://registry.npmjs.org/ktx-parse/-/ktx-parse-0.4.5.tgz" @@ -3187,16 +3191,17 @@ three-mesh-bvh@^0.5.10: integrity sha512-IMNHrAnsLCIxcFmAGkA4Wibw1QEpFQlkR72XUxZFOatNSpfMRUhJXQwQ5jPxbrX0W+OR838t/IR3laMOvQnT/g== three-stdlib@^2.10.2: - version "2.12.1" - resolved "https://registry.npmjs.org/three-stdlib/-/three-stdlib-2.12.1.tgz" - integrity sha512-G3SSsCOBiWa0sjjPt+K28ikQ84Plm/ZVUozMfWagK59kZqBWcaPVXpOThkAgvdBpm2zCWLW3edAoW/4XIbljVQ== + version "2.23.10" + resolved "https://registry.yarnpkg.com/three-stdlib/-/three-stdlib-2.23.10.tgz#94907a558a00da327bd74308c92078fea72f77fc" + integrity sha512-y0DlxaN5HZXI9hKjEtqO2xlCEt7XyDCOMvD2M3JJFBmYjwbU+PbJ1n3Z+7Hr/6BeVGE6KZYcqPMnfKrTK5WTJg== dependencies: - "@babel/runtime" "^7.16.7" - "@webgpu/glslang" "^0.0.15" + "@types/draco3d" "^1.4.0" + "@types/offscreencanvas" "^2019.6.4" + "@types/webxr" "^0.5.2" chevrotain "^10.1.2" draco3d "^1.4.1" fflate "^0.6.9" - ktx-parse "^0.2.1" + ktx-parse "^0.4.5" mmd-parser "^1.0.4" opentype.js "^1.3.3" potpack "^1.0.1" From 111686c75e82a2847197aa9f9c0b9575fc13d4ce Mon Sep 17 00:00:00 2001 From: Artem Ibatullin Date: Thu, 6 Jul 2023 10:02:34 +0300 Subject: [PATCH 2/2] fix review comments --- src/OculusHandModel.ts | 11 ++++++++--- src/XRHandMeshModel.ts | 1 + 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/OculusHandModel.ts b/src/OculusHandModel.ts index d6d501ef..717daf8e 100644 --- a/src/OculusHandModel.ts +++ b/src/OculusHandModel.ts @@ -53,6 +53,13 @@ class OculusHandModel extends Object3D { } private _onDisconnected: EventListener> = () => { + if (!this.xrInputSource?.hand) { + return; + } + this.motionControllerCleanup() + } + + private motionControllerCleanup(): void { this.clear() this.motionController?.dispose() this.motionController = null @@ -100,9 +107,7 @@ class OculusHandModel extends Object3D { } dispose(): void { - this.clear() - this.motionController?.dispose() - this.motionController = null + this.motionControllerCleanup() this.controller.removeEventListener('connected', this._onConnected) this.controller.removeEventListener('disconnected', this._onDisconnected) diff --git a/src/XRHandMeshModel.ts b/src/XRHandMeshModel.ts index 6ffb2ee4..e00c80df 100644 --- a/src/XRHandMeshModel.ts +++ b/src/XRHandMeshModel.ts @@ -95,6 +95,7 @@ class XRHandMeshModel { } } + // Hide hand mesh if all joints are invisible in case hand loses tracking if (allInvisible && this.scene) { this.scene.visible = false } else if (this.scene) {