Editor client: Pause/Resume video on tab visibility change

crates/lgn-graphics-renderer/src/components/render_surface.rs

@@ -127,6 +127,11 @@ impl SizeDependentResources {
     }
 }
 
+pub enum RenderSurfacePresentingStatus {
+    Presenting,
+    Paused,
+}
+
 #[derive(Component)]
 pub struct RenderSurface {
     id: RenderSurfaceId,
@@ -141,6 +146,7 @@ pub struct RenderSurface {
     debug_renderpass: Arc<RwLock<DebugRenderPass>>,
     egui_renderpass: Arc<RwLock<EguiPass>>,
     final_resolve_render_pass: Arc<RwLock<FinalResolveRenderPass>>,
+    presenting_status: RenderSurfacePresentingStatus,
 }
 
 impl RenderSurface {
@@ -263,7 +269,17 @@ impl RenderSurface {
         &self.resources.hzb_surface
     }
 
+    /// Call the `present` method of all the registered presenters.
+    /// No op if the render surface is "paused", i.e., it's `presenting`
+    /// attribute is `false`.
     pub fn present(&mut self, render_context: &RenderContext<'_>) {
+        if matches!(
+            self.presenting_status,
+            RenderSurfacePresentingStatus::Paused
+        ) {
+            return;
+        }
+
         let mut presenters = std::mem::take(&mut self.presenters);
 
         for presenter in &mut presenters {
@@ -287,6 +303,17 @@ impl RenderSurface {
     pub fn presenter_sem(&self) -> &Semaphore {
         &self.presenter_sems[self.render_frame_idx]
     }
+
+    pub fn pause(&mut self) -> &mut Self {
+        self.presenting_status = RenderSurfacePresentingStatus::Paused;
+        self
+    }
+
+    pub fn resume(&mut self) -> &mut Self {
+        self.presenting_status = RenderSurfacePresentingStatus::Presenting;
+        self
+    }
+
     fn new_with_id(
         id: RenderSurfaceId,
         renderer: &Renderer,
@@ -313,6 +340,7 @@ impl RenderSurface {
                 pipeline_manager,
             ))),
             presenters: Vec::new(),
+            presenting_status: RenderSurfacePresentingStatus::Presenting,
         }
     }
 }

crates/lgn-streamer/src/lib.rs

@@ -49,7 +49,9 @@ impl Plugin for StreamerPlugin {
             .init_resource::<streamer::control_stream::ControlStreams>()
             .insert_resource(StreamEncoder::new(self.config.enable_hw_encoding))
             .add_event::<streamer::VideoStreamEvent>()
+            .add_event::<streamer::ControlEvent>()
             .add_system(streamer::handle_stream_events)
+            .add_system(streamer::handle_control_events)
             .add_system(streamer::update_streams)
             .add_system(streamer::on_app_exit)
             .add_system(streamer::on_render_surface_created_for_window)

crates/lgn-streamer/src/streamer/events.rs

@@ -1,3 +1,7 @@
+#![allow(clippy::use_self)]
+
+use std::sync::Arc;
+
 use anyhow::bail;
 use lgn_input::{
     keyboard::KeyboardInput,
@@ -8,6 +12,29 @@ use lgn_input::{
 use lgn_math::Vec2;
 use lgn_window::WindowId;
 use serde::Deserialize;
+use webrtc::data_channel::RTCDataChannel;
+
+pub(crate) struct ControlEvent {
+    #[allow(unused)]
+    pub(crate) window_id: WindowId,
+    pub(crate) info: ControlEventInfo,
+    #[allow(unused)]
+    pub(crate) control_data_channel: Arc<RTCDataChannel>,
+}
+
+impl ControlEvent {
+    pub(crate) fn parse(
+        window_id: WindowId,
+        control_data_channel: Arc<RTCDataChannel>,
+        data: &[u8],
+    ) -> anyhow::Result<Self> {
+        Ok(Self {
+            window_id,
+            info: serde_json::from_slice(data)?,
+            control_data_channel,
+        })
+    }
+}
 
 pub(crate) struct VideoStreamEvent {
     pub(crate) window_id: WindowId,
@@ -101,14 +128,23 @@ pub(crate) enum Input {
     KeyboardInput(KeyboardInput),
 }
 
+#[derive(Debug, Deserialize)]
+#[serde(tag = "event")]
+pub(crate) enum ControlEventInfo {
+    #[serde(rename = "pause")]
+    Pause,
+    #[serde(rename = "resume")]
+    Resume,
+}
+
 #[derive(Debug, Deserialize)]
 #[serde(tag = "event")]
 pub(crate) enum VideoStreamEventInfo {
     #[serde(rename = "resize")]
     Resize { width: u32, height: u32 },
     #[serde(rename = "initialize")]
-    #[allow(dead_code)]
     Initialize {
+        #[allow(dead_code)]
         color: Color,
         width: u32,
         height: u32,

crates/lgn-streamer/src/streamer/mod.rs

@@ -106,9 +106,11 @@ impl Streamer {
     }
 }
 
+#[allow(clippy::too_many_arguments)]
 pub(crate) fn handle_stream_events(
     async_rt: Res<'_, TokioAsyncRuntime>,
     streamer: Res<'_, Streamer>,
+    mut control_events: EventWriter<'_, '_, ControlEvent>,
     mut video_stream_events: EventWriter<'_, '_, VideoStreamEvent>,
     mut window_list: ResMut<'_, Windows>,
     mut streamer_windows: ResMut<'_, StreamerWindows>,
@@ -184,12 +186,39 @@ pub(crate) fn handle_stream_events(
                     window_id,
                 );
             }
-            StreamEvent::ControlChannelMessageReceived(_, _, _) => {
-                //commands
-                //    .entity(stream_id.entity)
-                //    .get_components(|stream: &mut ControlStream| {});
+            StreamEvent::ControlChannelMessageReceived(stream_id, data_channel, msg) => {
+                match ControlEvent::parse(stream_id, data_channel, &msg.data) {
+                    Ok(event) => {
+                        control_events.send(event);
+                    }
+                    Err(e) => {
+                        warn!("Ignoring unknown video data channel message: {}", e);
+                    }
+                }
+            }
+        }
+    }
+}
+
+pub(crate) fn handle_control_events(
+    mut control_events: EventReader<'_, '_, ControlEvent>,
+    mut render_surfaces: Query<'_, '_, &mut RenderSurface>,
+) {
+    for event in control_events.iter() {
+        match &event.info {
+            ControlEventInfo::Pause => {
+                trace!("Received control Pause event, pausing stream");
 
-                //control_stream.parse_and_append(msg);
+                render_surfaces.for_each_mut(|mut render_surface| {
+                    render_surface.pause();
+                });
+            }
+            ControlEventInfo::Resume => {
+                trace!("Received control Resume event, resuming stream");
+
+                render_surfaces.for_each_mut(|mut render_surface| {
+                    render_surface.resume();
+                });
             }
         }
     }

crates/lgn-web-client/frontend/src/actions/videoPlayer.ts

@@ -19,6 +19,11 @@ export type PushableHTMLVideoElement = HTMLVideoElement & {
   push(data: ArrayBuffer): void;
 };
 
+export type Options = {
+  controlChannel: RTCDataChannel | null;
+  onFatal?: () => void;
+};
+
 /**
  * Takes an `HTMLVideoElement` and attach a `push` method to it.
  *
@@ -29,7 +34,7 @@ export type PushableHTMLVideoElement = HTMLVideoElement & {
  */
 export default function videoPlayer(
   videoElement: HTMLVideoElement,
-  options?: { onFatal?: () => void }
+  options: Options
 ) {
   videoElement.muted = true;
   videoElement.disablePictureInPicture = true;
@@ -41,6 +46,39 @@ export default function videoPlayer(
   let listeners: Listener[] = [];
   let lastFrameId = -1;
 
+  function onVisibilityChange() {
+    switch (document.visibilityState) {
+      case "hidden": {
+        videoElement.pause();
+
+        options.controlChannel?.send(
+          JSON.stringify({
+            event: "pause",
+          })
+        );
+
+        break;
+      }
+
+      case "visible": {
+        videoElement.play().catch(() => {
+          log.warn(
+            "video",
+            "Video player's pause method was called while the play method was running"
+          );
+        });
+
+        options.controlChannel?.send(
+          JSON.stringify({
+            event: "resume",
+          })
+        );
+
+        break;
+      }
+    }
+  }
+
   function addListener(source: Source, name: string, f: () => void) {
     source.addEventListener(name, f);
 
@@ -85,7 +123,7 @@ export default function videoPlayer(
       } catch (error) {
         log.error("video", error);
 
-        options?.onFatal && options.onFatal();
+        options.onFatal && options.onFatal();
 
         destroy();
       }
@@ -120,6 +158,8 @@ export default function videoPlayer(
     }
   }
 
+  document.addEventListener("visibilitychange", onVisibilityChange);
+
   (videoElement as PushableHTMLVideoElement).push = (data) => {
     const chunk = new Uint8Array(data);
     const frameId =
@@ -154,4 +194,15 @@ export default function videoPlayer(
       lastFrameId = frameId;
     }
   };
+
+  return {
+    destroy() {
+      document.removeEventListener("visibilitychange", onVisibilityChange);
+
+      destroy();
+    },
+    update({ controlChannel: newControlChannel }: Options) {
+      options.controlChannel = newControlChannel;
+    },
+  };
 }

crates/lgn-web-client/frontend/src/components/RemoteWindow.svelte

@@ -99,7 +99,7 @@
 
     controlChannel = peerConnection.createDataChannel("control");
 
-    videoChannel.binaryType = "arraybuffer";
+    controlChannel.binaryType = "arraybuffer";
 
     peerConnection.onnegotiationneeded = async () => {
       if (!peerConnection) {
@@ -329,7 +329,11 @@
   use:resize={onVideoResize}
   use:remoteWindowInputs={onRemoteWindowInput}
 >
-  <video class="video" use:videoPlayer bind:this={videoElement}>
+  <video
+    class="video"
+    use:videoPlayer={{ controlChannel }}
+    bind:this={videoElement}
+  >
     <track kind="captions" />
   </video>
   <!-- TODO: Set opacity to 70 or so to still see the video player, blinks for the moment -->