Full-Resolution Camera Downloads

README.md

@@ -54,7 +54,7 @@ The rover can be operated through Mission Control with either a keyboard or two
 ![Armo controls](/src/components/help/armControls.png)
 ![Keyboard controls](/src/components/help/keyboardControls.png)
 
-## Messages (`v2024.1.2`)
+## Messages (`v2024.2.0`)
 The JSON objects sent between Mission Control and the rover server are termed *messages*. Each message has a type property and a number of additional parameters depending on the type. The usage of each type of message is detailed below.
 
 ## Mounted Peripheral Report
@@ -380,6 +380,38 @@ Sent from the rover server to inform Mission Control of a single frame of a came
 - `camera` - the name of the camera: `mast|hand|wrist`
 - `data` - the raw h264 frame data, or `null` if no data is available
 
+## Camera Frame Request
+### Description
+Sent from Mission Control to instruct the rover server to send a Camera Frame Report.
+
+### Syntax
+```
+{
+  type: "cameraFrameRequest",
+  camera: string
+}
+```
+
+### Parameters
+- `camera` - the name of the camera
+
+## Camera Frame Report
+### Description
+Sent from the rover server to inform Mission Control of a full resolution lossless camera frame.
+
+### Syntax
+```
+{
+  type: "cameraFrameReport",
+  camera: string,
+  data: string | null
+}
+```
+
+### Parameters
+- `camera` - the name of the camera
+- `data` - the image, base64 encoded
+
 
 ## Autonomous Waypoint Navigation Request
 ### Description

src/components/camera/CameraStream.jsx

@@ -4,29 +4,28 @@ import JMuxer from "jmuxer";
 import {
   openCameraStream,
   closeCameraStream,
-  selectCameraStreamFrameData
+  selectCameraStreamFrameData,
+  requestCameraFrame,
 } from "../../store/camerasSlice";
+import { selectRoverIsConnected } from "../../store/roverSocketSlice";
 import camelCaseToTitle from "../../util/camelCaseToTitle";
 import "./CameraStream.css";
-
 /**
  * Takes:
  *    cameraTitle: the camera title,
  *    cameraName: the camera name,
  *    unloadCallback: a callback that is ran before the window is fully unloaded
- *    video_width: the stream width
- *    video_height: the stream height
+ *    downloadCallback: a callback that is ran when the download button is pressed
  * Returns: Promise of an object with keys: window, canvas, context, aspectRatio
  */
-async function createPopOutWindow(cameraTitle, cameraName, unloadCallback, video_width, video_height) {
+async function createPopOutWindow(cameraTitle, cameraName, unloadCallback, downloadCallback) {
   let newWindow = window.open("/camera/cam_popout.htm", "", "width=500,height=500");
 
   const returnPromise = new Promise((resolve, reject) => {
     newWindow.onload = () => {
       newWindow.document.title = `${cameraTitle} Stream`;
       newWindow.document.querySelector('#ext-title').innerText = cameraTitle;
-      newWindow.document.querySelector('#ext-download-button').setAttribute("onclick", `download("${cameraTitle}", ${video_width}, ${video_height})`);
-
+      newWindow.document.querySelector('#ext-download-button').onclick = downloadCallback;
       let canvas = newWindow.document.querySelector('#ext-vid');
       let context = canvas.getContext('2d');
       let aspectRatio = document.querySelector(`#${cameraName}-player`).videoHeight / document.querySelector(`#${cameraName}-player`).videoWidth;
@@ -37,7 +36,7 @@ async function createPopOutWindow(cameraTitle, cameraName, unloadCallback, video
 
       window.onunload = () => {
         if (newWindow && !newWindow.closed) {
-            newWindow.close();
+          newWindow.close();
         }
       };
 
@@ -49,43 +48,13 @@ async function createPopOutWindow(cameraTitle, cameraName, unloadCallback, video
         context: context,
         aspectRatio: aspectRatio
       };
-      resolve(output); 
+      resolve(output);
     }
   });
 
   return returnPromise;
 }
 
-/**
- * Takes
- *    video: HTMLVideoElement representing the video tag which should be processed.
- *    cameraTitle: name of the camera, used for the filename.
- * Note: This is the React version of this function for this CameraStream component,
- *    The popout window "download" button uses a seperate function, defined in 
- *    /public/camera/cam_popout.js. If you make changes to this function, you need to 
- *    make corresponding changes to the cam_popout.js file.
- */
-function downloadCurrentFrame(video, cameraTitle) {
-  if (!video || !(video.videoWidth && video.videoHeight)) return null;
-  let canvas = document.createElement('canvas');
-  let context = canvas.getContext('2d');
-  canvas.width = video.videoWidth;
-  canvas.height = video.videoHeight;
-  context.drawImage(video, 0, 0, canvas.width, canvas.height);
-
-  let link = document.createElement("a");
-  link.href = canvas.toDataURL("image/jpeg", 1);
-
-  let time = new Date();
-  let timezoneOffset = time.getTimezoneOffset() * 60000;
-  let timeString = new Date(time - timezoneOffset).toISOString().replace(":", "_").substring(0, 19);
-
-  link.download = `${cameraTitle}-${timeString}.jpg`;
-  document.body.appendChild(link);
-  link.click();
-  document.body.removeChild(link);
-}
-
 function CameraStream({ cameraName }) {
   const dispatch = useDispatch();
   useEffect(() => {
@@ -97,6 +66,7 @@ function CameraStream({ cameraName }) {
     };
   }, [cameraName, dispatch]);
 
+  const roverIsConnected = useSelector(selectRoverIsConnected);
   const frameDataArray = useSelector(selectCameraStreamFrameData(cameraName));
   const cameraTitle = camelCaseToTitle(cameraName);
   const [hasRendered, setHasRendered] = useState(false);
@@ -107,17 +77,22 @@ function CameraStream({ cameraName }) {
   const [popoutWindow, setPopoutWindow] = useState(null);
   const [aspectRatio, setAspectRatio] = useState(1);
 
+
   const cameraCanvas = useRef(null);  // used for popout window
   const cameraContext = useRef(null);  // used for popout window
-  
+
   const vidTag = useMemo(() => {
-    return <video style={{opacity: popoutWindow ? '0' : '1'}} id={`${cameraName}-player`} className='video-tag' muted autoPlay preload="auto" alt={`${cameraTitle} stream`}></video>;
-  }, [cameraName, cameraTitle, popoutWindow])
-
+    return <video style={{ opacity: popoutWindow ? '0' : '1' }} id={`${cameraName}-player`} className='video-tag' muted autoPlay preload="auto" alt={`${cameraTitle} stream`}></video>;
+  }, [cameraName, cameraTitle, popoutWindow]);
+
+  const requestDownloadFrame = useCallback(() => {
+    dispatch(requestCameraFrame({ cameraName }));
+  }, [cameraName, dispatch]);
+
   const drawFrameOnExt = useCallback((window, last_ww, last_wh) => {
     if (vidTag && window && cameraCanvas) {
       // draw it onto the popout window
-      
+
       if (window.innerWidth !== last_ww || window.innerHeight !== last_wh) {
         // if the window is wider than the stream
         if (window.innerHeight / window.innerWidth > aspectRatio) {
@@ -131,18 +106,20 @@ function CameraStream({ cameraName }) {
         }
       }
 
-      if (hasFrame) {
-        let button = window.document.querySelector("#ext-download-button");
-        button.removeAttribute('disabled');
-      }
-
       let video = document.querySelector(`#${vidTag.props.id}`);
       cameraContext.current.drawImage(video, 0, 0, cameraCanvas.current.width, cameraCanvas.current.height);
       last_ww = window.innerWidth;
       last_wh = window.innerHeight;
       window.requestAnimationFrame(() => { drawFrameOnExt(window, last_ww, last_wh); });
     }
-  }, [vidTag, aspectRatio, hasFrame]);
+  }, [vidTag, aspectRatio]);
+
+  useEffect(() => {
+    if (popoutWindow) {
+      let button = popoutWindow.document.querySelector("#ext-download-button");
+      if (button) button.disabled = !(hasFrame && roverIsConnected);
+    }
+  }, [popoutWindow, hasFrame, roverIsConnected]);
 
   const handlePopOut = useCallback(async () => {
     if (popoutWindow) {
@@ -152,7 +129,7 @@ function CameraStream({ cameraName }) {
     } else if (vidTag) {
       // if the window popout doesn't exist
       let video = document.getElementById(vidTag.props.id);
-      let { popout, canvas, context, aspectRatio } = await createPopOutWindow(cameraTitle, cameraName, () => setPopoutWindow(null), video.videoWidth, video.videoHeight);
+      let { popout, canvas, context, aspectRatio } = await createPopOutWindow(cameraTitle, cameraName, () => setPopoutWindow(null), requestDownloadFrame);
       setAspectRatio(aspectRatio);
       setPopoutWindow(popout);
       cameraCanvas.current = canvas;
@@ -169,10 +146,10 @@ function CameraStream({ cameraName }) {
         flushingTime: 0,
         maxDelay: 50,
         clearBuffer: true,
-        onError: function(data) {
+        onError: function (data) {
           console.warn('Buffer error encountered', data);
         },
-        
+
         onMissingVideoFrames: function (data) {
           console.warn('Video frames missing', data);
         }
@@ -216,7 +193,7 @@ function CameraStream({ cameraName }) {
       }
     };
   }, [popoutWindow]);
-  
+
   useEffect(() => {
     // this indicates that the site has rendered and the player is able to be modified (specifically the src)
     setHasRendered(true);
@@ -225,20 +202,20 @@ function CameraStream({ cameraName }) {
   return (
     <div className="camera-stream">
       <h2 className="camera-stream__camera-name">{cameraTitle}</h2>
-      <div className="video-container">{ vidTag }</div>
-      { popoutWindow ? <h3>Stream In External Window</h3> : (!frameDataArray && <h3>No Stream Available</h3>) }
+      <div className="video-container">{vidTag}</div>
+      {popoutWindow ? <h3>Stream In External Window</h3> : (!frameDataArray && <h3>No Stream Available</h3>)}
       <div className='camera-stream-fps'>FPS: {currentFpsAvg && frameDataArray ? Math.round(currentFpsAvg) : 'N/A'}</div>
       <div className='camera-stream-pop-header'>
         <span className='camera-stream-pop-button'
-        title={`Open "${cameraTitle}" camera stream in a new window.`}
-        onClick={ handlePopOut }>
-          { popoutWindow ? 'Merge Window' : 'Pop Out' }
+          title={`Open "${cameraTitle}" camera stream in a new window.`}
+          onClick={handlePopOut}>
+          {popoutWindow ? 'Merge Window' : 'Pop Out'}
         </span>
       </div>
       <div className='camera-stream-download-header'>
         <button className='camera-stream-download-button'
-        title={`Download "${cameraTitle}" camera stream current frame`}
-        onClick={() => downloadCurrentFrame(document.querySelector(`#${vidTag.props.id}`), cameraTitle)} disabled={!hasFrame}>
+          title={`Download "${cameraTitle}" camera stream current frame`}
+          onClick={requestDownloadFrame} disabled={!(hasFrame && roverIsConnected)}>
           Download
         </button>
       </div>

src/store/camerasSlice.js

@@ -25,6 +25,8 @@ const camerasSlice = createSlice({
       state[cameraName].frameData = null;
     },
 
+    requestCameraFrame() { },
+
     cameraStreamDataReportReceived(state, action) {
       const { cameraName, frameData } = action.payload;
       if (state[cameraName].isStreaming)
@@ -36,7 +38,8 @@ const camerasSlice = createSlice({
 export const {
   openCameraStream,
   closeCameraStream,
-  cameraStreamDataReportReceived
+  cameraStreamDataReportReceived,
+  requestCameraFrame
 } = camerasSlice.actions;
 
 export const selectAllCameraNames = state => Object.keys(state.cameras);

src/store/middleware/camerasMiddleware.js

@@ -2,13 +2,16 @@ import {
   openCameraStream,
   closeCameraStream,
   cameraStreamDataReportReceived,
+  requestCameraFrame,
 } from "../camerasSlice";
 import {
   messageReceivedFromRover,
   messageRover,
   roverConnected,
   roverDisconnected
 } from "../roverSocketSlice";
+import camelCaseToTitle from "../../util/camelCaseToTitle";
+
 
 /**
  * Middleware that handles requesting and receiving camera streams from the
@@ -39,6 +42,16 @@ const camerasMiddleware = store => next => action => {
       break;
     }
 
+    case requestCameraFrame.type: {
+      store.dispatch(messageRover({
+        message: {
+          type: "cameraFrameRequest",
+          camera: action.payload.cameraName
+        }
+      }));
+      break;
+    }
+
     case roverConnected.type: {
       // Inform the rover of camera streams we would like to receive when we
       // connect.
@@ -74,11 +87,26 @@ const camerasMiddleware = store => next => action => {
 
     case messageReceivedFromRover.type: {
       const { message } = action.payload;
-      if (message.type === "cameraStreamReport")
+      if (message.type === "cameraStreamReport") {
         store.dispatch(cameraStreamDataReportReceived({
           cameraName: message.camera,
           frameData: message.data
         }));
+      } else if (message.type === "cameraFrameReport") {
+        if (message.data !== "") {
+          let link = document.createElement("a");
+          link.href = `data:image/jpg;base64,${message.data}`;
+          let time = new Date();
+          let timezoneOffset = time.getTimezoneOffset() * 60000;
+          let timeString = new Date(time - timezoneOffset).toISOString().replace(":", "_").substring(0, 19);
+
+          link.download = `${camelCaseToTitle(message.camera)}-${timeString}.jpg`;
+          document.body.appendChild(link);
+          link.click();
+          document.body.removeChild(link);
+
+        }
+      }
       break;
     }