From 1b3d1c7271638f346e42d5455e84237f55eeda7e Mon Sep 17 00:00:00 2001 From: zmerp Date: Sun, 19 Jan 2025 18:32:26 +0100 Subject: [PATCH] =?UTF-8?q?fix(client=5Fopenxr):=20=F0=9F=90=9B=20Try=20fi?= =?UTF-8?q?xing=20crash=20on=20Quest=202=20related=20to=20multimodal=20(#2?= =?UTF-8?q?654)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(client_openxr): 🐛 Try fixing crash on Quest 2 related to multimodal * Address comments --- .../extra_extensions/eye_gaze_interaction.rs | 17 +- .../client_openxr/src/extra_extensions/mod.rs | 20 ++ .../src/extra_extensions/multimodal_input.rs | 182 +++++++++++------- alvr/client_openxr/src/interaction.rs | 31 +-- alvr/client_openxr/src/lib.rs | 3 +- 5 files changed, 157 insertions(+), 96 deletions(-) diff --git a/alvr/client_openxr/src/extra_extensions/eye_gaze_interaction.rs b/alvr/client_openxr/src/extra_extensions/eye_gaze_interaction.rs index cb1798509f..36b447dd12 100644 --- a/alvr/client_openxr/src/extra_extensions/eye_gaze_interaction.rs +++ b/alvr/client_openxr/src/extra_extensions/eye_gaze_interaction.rs @@ -1,27 +1,12 @@ use openxr::{self as xr, sys}; use std::ptr; -fn get_props(session: &xr::Session, system: xr::SystemId, default_struct: T) -> Option { - let instance = session.instance(); - - let mut props = default_struct; - let mut system_properties = sys::SystemProperties::out((&mut props as *mut T).cast()); - let result = unsafe { - (instance.fp().get_system_properties)( - instance.as_raw(), - system, - system_properties.as_mut_ptr(), - ) - }; - (result.into_raw() >= 0).then_some(props) -} - pub fn supports_eye_gaze_interaction(session: &xr::Session, system: xr::SystemId) -> bool { if session.instance().exts().ext_eye_gaze_interaction.is_none() { return false; } - get_props( + super::get_props( session, system, sys::SystemEyeGazeInteractionPropertiesEXT { diff --git a/alvr/client_openxr/src/extra_extensions/mod.rs b/alvr/client_openxr/src/extra_extensions/mod.rs index 0f08f9f709..6b261f5440 100644 --- a/alvr/client_openxr/src/extra_extensions/mod.rs +++ b/alvr/client_openxr/src/extra_extensions/mod.rs @@ -25,3 +25,23 @@ fn xr_res(result: sys::Result) -> xr::Result<()> { Err(result) } } + +fn get_props( + session: &xr::Session, + system: xr::SystemId, + default_struct: T, +) -> xr::Result { + let instance = session.instance(); + + let mut props = default_struct; + let mut system_properties = sys::SystemProperties::out((&mut props as *mut T).cast()); + let result = unsafe { + (instance.fp().get_system_properties)( + instance.as_raw(), + system, + system_properties.as_mut_ptr(), + ) + }; + + xr_res(result).map(|_| props) +} diff --git a/alvr/client_openxr/src/extra_extensions/multimodal_input.rs b/alvr/client_openxr/src/extra_extensions/multimodal_input.rs index 4cc97f8c5a..a6076be907 100644 --- a/alvr/client_openxr/src/extra_extensions/multimodal_input.rs +++ b/alvr/client_openxr/src/extra_extensions/multimodal_input.rs @@ -12,87 +12,139 @@ pub const META_SIMULTANEOUS_HANDS_AND_CONTROLLERS_EXTENSION_NAME: &str = "XR_META_simultaneous_hands_and_controllers"; pub const META_DETACHED_CONTROLLERS_EXTENSION_NAME: &str = "XR_META_detached_controllers"; +static TYPE_SYSTEM_SIMULTANEOUS_HANDS_AND_CONTROLLERS_PROPERTIES_META: Lazy = + Lazy::new(|| xr::StructureType::from_raw(1000532001)); static TYPE_SIMULTANEOUS_HANDS_AND_CONTROLLERS_TRACKING_RESUME_INFO_META: Lazy = Lazy::new(|| xr::StructureType::from_raw(1000532002)); static TYPE_SIMULTANEOUS_HANDS_AND_CONTROLLERS_TRACKING_PAUSE_INFO_META: Lazy = Lazy::new(|| xr::StructureType::from_raw(1000532003)); #[repr(C)] -pub struct SimultaneousHandsAndControllersTrackingResumeInfoMETA { +struct SystemSymultaneousHandsAndControllersPropertiesMETA { + ty: xr::StructureType, + next: *const c_void, + supports_simultaneous_hands_and_controllers: sys::Bool32, +} + +#[repr(C)] +struct SimultaneousHandsAndControllersTrackingResumeInfoMETA { ty: xr::StructureType, next: *const c_void, } #[repr(C)] -pub struct SimultaneousHandsAndControllersTrackingPauseInfoMETA { +struct SimultaneousHandsAndControllersTrackingPauseInfoMETA { ty: xr::StructureType, next: *const c_void, } -pub type ResumeSimultaneousHandsAndControllersTrackingMETA = - unsafe extern "system" fn( - sys::Session, - *const SimultaneousHandsAndControllersTrackingResumeInfoMETA, - ) -> sys::Result; -pub type PauseSimultaneousHandsAndControllersTrackingMETA = - unsafe extern "system" fn( - sys::Session, - *const SimultaneousHandsAndControllersTrackingPauseInfoMETA, - ) -> sys::Result; - -pub fn resume_simultaneous_hands_and_controllers_tracking_meta( - session: &xr::Session, -) -> xr::Result<()> { - let resume_simultaneous_hands_and_controllers_tracking_meta = unsafe { - let mut resume_simultaneous_hands_and_controllers_tracking_meta = None; - let _ = (session.instance().fp().get_instance_proc_addr)( - session.instance().as_raw(), - c"xrResumeSimultaneousHandsAndControllersTrackingMETA".as_ptr(), - &mut resume_simultaneous_hands_and_controllers_tracking_meta, - ); - - resume_simultaneous_hands_and_controllers_tracking_meta.map(|pfn| { - mem::transmute::(pfn) - }) - } - .ok_or(sys::Result::ERROR_EXTENSION_NOT_PRESENT)?; - - let resume_info = SimultaneousHandsAndControllersTrackingResumeInfoMETA { - ty: *TYPE_SIMULTANEOUS_HANDS_AND_CONTROLLERS_TRACKING_RESUME_INFO_META, - next: ptr::null(), - }; - unsafe { - super::xr_res(resume_simultaneous_hands_and_controllers_tracking_meta( - session.as_raw(), - &resume_info, - )) - } +type ResumeSimultaneousHandsAndControllersTrackingMETA = unsafe extern "system" fn( + sys::Session, + *const SimultaneousHandsAndControllersTrackingResumeInfoMETA, +) -> sys::Result; +type PauseSimultaneousHandsAndControllersTrackingMETA = unsafe extern "system" fn( + sys::Session, + *const SimultaneousHandsAndControllersTrackingPauseInfoMETA, +) -> sys::Result; + +pub struct MultimodalMeta { + session: xr::Session, + resume_simultaneous_hands_and_controllers_tracking_meta: + ResumeSimultaneousHandsAndControllersTrackingMETA, + pause_simultaneous_hands_and_controllers_tracking_meta: + PauseSimultaneousHandsAndControllersTrackingMETA, } -pub fn pause_simultaneous_hands_and_controllers_tracking_meta( - session: &xr::Session, -) -> xr::Result<()> { - let pause_simultaneous_hands_and_controllers_tracking_meta = unsafe { - let mut pause_simultaneous_hands_and_controllers_tracking_meta = None; - let _ = (session.instance().fp().get_instance_proc_addr)( - session.instance().as_raw(), - c"xrPauseSimultaneousHandsAndControllersTrackingMETA".as_ptr(), - &mut pause_simultaneous_hands_and_controllers_tracking_meta, - ); - - pause_simultaneous_hands_and_controllers_tracking_meta.map(|pfn| { - mem::transmute::(pfn) - }) +impl MultimodalMeta { + pub fn new( + session: xr::Session, + extra_extensions: &[String], + system: xr::SystemId, + ) -> xr::Result { + if !extra_extensions + .contains(&META_SIMULTANEOUS_HANDS_AND_CONTROLLERS_EXTENSION_NAME.to_owned()) + || !extra_extensions.contains(&META_DETACHED_CONTROLLERS_EXTENSION_NAME.to_owned()) + { + return Err(sys::Result::ERROR_EXTENSION_NOT_PRESENT); + } + + let resume_simultaneous_hands_and_controllers_tracking_meta = unsafe { + let mut resume_simultaneous_hands_and_controllers_tracking_meta = None; + let _ = (session.instance().fp().get_instance_proc_addr)( + session.instance().as_raw(), + c"xrResumeSimultaneousHandsAndControllersTrackingMETA".as_ptr(), + &mut resume_simultaneous_hands_and_controllers_tracking_meta, + ); + + resume_simultaneous_hands_and_controllers_tracking_meta.map(|pfn| { + mem::transmute::( + pfn, + ) + }) + } + .ok_or(sys::Result::ERROR_EXTENSION_NOT_PRESENT)?; + + let pause_simultaneous_hands_and_controllers_tracking_meta = unsafe { + let mut pause_simultaneous_hands_and_controllers_tracking_meta = None; + let _ = (session.instance().fp().get_instance_proc_addr)( + session.instance().as_raw(), + c"xrPauseSimultaneousHandsAndControllersTrackingMETA".as_ptr(), + &mut pause_simultaneous_hands_and_controllers_tracking_meta, + ); + + pause_simultaneous_hands_and_controllers_tracking_meta.map(|pfn| { + mem::transmute::( + pfn, + ) + }) + } + .ok_or(sys::Result::ERROR_EXTENSION_NOT_PRESENT)?; + + let props = super::get_props( + &session, + system, + SystemSymultaneousHandsAndControllersPropertiesMETA { + ty: *TYPE_SYSTEM_SIMULTANEOUS_HANDS_AND_CONTROLLERS_PROPERTIES_META, + next: std::ptr::null(), + supports_simultaneous_hands_and_controllers: xr::sys::FALSE, + }, + )?; + + if props.supports_simultaneous_hands_and_controllers.into() { + Ok(Self { + session: session.into_any_graphics(), + resume_simultaneous_hands_and_controllers_tracking_meta, + pause_simultaneous_hands_and_controllers_tracking_meta, + }) + } else { + Err(sys::Result::ERROR_FEATURE_UNSUPPORTED) + } + } + + pub fn resume(&self) -> xr::Result<()> { + let resume_info = SimultaneousHandsAndControllersTrackingResumeInfoMETA { + ty: *TYPE_SIMULTANEOUS_HANDS_AND_CONTROLLERS_TRACKING_RESUME_INFO_META, + next: ptr::null(), + }; + unsafe { + super::xr_res((self + .resume_simultaneous_hands_and_controllers_tracking_meta)( + self.session.as_raw(), + &resume_info, + )) + } } - .ok_or(sys::Result::ERROR_EXTENSION_NOT_PRESENT)?; - - let pause_info = SimultaneousHandsAndControllersTrackingPauseInfoMETA { - ty: *TYPE_SIMULTANEOUS_HANDS_AND_CONTROLLERS_TRACKING_PAUSE_INFO_META, - next: ptr::null(), - }; - unsafe { - super::xr_res(pause_simultaneous_hands_and_controllers_tracking_meta( - session.as_raw(), - &pause_info, - )) + + pub fn pause(&self) -> xr::Result<()> { + let pause_info = SimultaneousHandsAndControllersTrackingPauseInfoMETA { + ty: *TYPE_SIMULTANEOUS_HANDS_AND_CONTROLLERS_TRACKING_PAUSE_INFO_META, + next: ptr::null(), + }; + unsafe { + super::xr_res((self + .pause_simultaneous_hands_and_controllers_tracking_meta)( + self.session.as_raw(), + &pause_info, + )) + } } } diff --git a/alvr/client_openxr/src/interaction.rs b/alvr/client_openxr/src/interaction.rs index 6d2901c1d8..e9c2eaa17e 100644 --- a/alvr/client_openxr/src/interaction.rs +++ b/alvr/client_openxr/src/interaction.rs @@ -1,6 +1,6 @@ use crate::{ extra_extensions::{ - self, BodyTrackerFB, EyeTrackerSocial, FaceTracker2FB, FacialTrackerHTC, + self, BodyTrackerFB, EyeTrackerSocial, FaceTracker2FB, FacialTrackerHTC, MultimodalMeta, BODY_JOINT_SET_FULL_BODY_META, FULL_BODY_JOINT_COUNT_META, FULL_BODY_JOINT_LEFT_FOOT_BALL_META, FULL_BODY_JOINT_LEFT_LOWER_LEG_META, FULL_BODY_JOINT_RIGHT_FOOT_BALL_META, FULL_BODY_JOINT_RIGHT_LOWER_LEG_META, @@ -108,6 +108,7 @@ pub struct InteractionContext { pub action_set: xr::ActionSet, pub button_actions: HashMap, pub hands_interaction: [HandInteraction; 2], + multimodal_handle: Option, pub multimodal_hands_enabled: bool, pub face_sources: FaceSources, pub body_sources: BodySources, @@ -116,9 +117,9 @@ pub struct InteractionContext { impl InteractionContext { pub fn new( xr_session: xr::Session, + extra_extensions: &[String], xr_system: xr::SystemId, platform: Platform, - supports_multimodal: bool, ) -> Self { let xr_instance = xr_session.instance(); @@ -229,9 +230,13 @@ impl InteractionContext { "/user/hand/right/output/haptic", )); + let multimodal_handle = create_ext_object("MultimodalMeta", Some(true), || { + MultimodalMeta::new(xr_session.clone(), extra_extensions, xr_system) + }); + let left_detached_controller_pose_action; let right_detached_controller_pose_action; - if supports_multimodal { + if multimodal_handle.is_some() { // Note: when multimodal input is enabled, both controllers and hands will always be active. // To be able to detect when controllers are actually held, we have to register detached // controllers pose; the controller pose will be diverted to the detached controllers when @@ -355,6 +360,7 @@ impl InteractionContext { skeleton_tracker: right_hand_tracker, }, ], + multimodal_handle, multimodal_hands_enabled: false, face_sources: FaceSources { combined_eyes_source, @@ -371,8 +377,9 @@ impl InteractionContext { pub fn select_sources(&mut self, config: &InteractionSourcesConfig) { // First of all, disable/delete all sources. This ensures there are no conflicts - extra_extensions::pause_simultaneous_hands_and_controllers_tracking_meta(&self.xr_session) - .ok(); + if let Some(handle) = &mut self.multimodal_handle { + handle.pause().ok(); + } self.multimodal_hands_enabled = false; self.face_sources.eye_tracker_fb = None; self.face_sources.face_tracker_fb = None; @@ -407,14 +414,12 @@ impl InteractionContext { // Note: We cannot enable multimodal if fb body tracking is active. It would result in a // ERROR_RUNTIME_FAILURE crash. - if config.body_tracking.is_none() - && config.prefers_multimodal_input - && extra_extensions::resume_simultaneous_hands_and_controllers_tracking_meta( - &self.xr_session, - ) - .is_ok() - { - self.multimodal_hands_enabled = true; + if config.body_tracking.is_none() && config.prefers_multimodal_input { + if let Some(handle) = &mut self.multimodal_handle { + if handle.resume().is_ok() { + self.multimodal_hands_enabled = true; + } + } } self.face_sources.eye_tracker_fb = create_ext_object( diff --git a/alvr/client_openxr/src/lib.rs b/alvr/client_openxr/src/lib.rs index 065fbd5a4c..719be4256d 100644 --- a/alvr/client_openxr/src/lib.rs +++ b/alvr/client_openxr/src/lib.rs @@ -267,10 +267,9 @@ pub fn entry_point() { let interaction_context = Arc::new(RwLock::new(InteractionContext::new( xr_session.clone(), + &exts.other, xr_system, platform, - exts.other - .contains(&META_SIMULTANEOUS_HANDS_AND_CONTROLLERS_EXTENSION_NAME.to_owned()), ))); let mut lobby = Lobby::new(