pv / rm depth ahat coop model

microsoft/HoloLens2ForCV#133

hl2ss/hl2ss/personal_video.cpp

@@ -105,6 +105,14 @@ void PersonalVideo_Initialize()
     PersonalVideo_FindVideoSource(g_mediaCapture, g_videoSource);
 }
 
+// OK
+void PersonalVideo_Close()
+{
+    g_videoSource = nullptr;
+    g_mediaCapture.Close();
+    g_mediaCapture = nullptr;
+}
+
 // OK
 bool PersonalVideo_SetFormat(uint32_t width, uint32_t height, uint32_t framerate)
 {

hl2ss/hl2ss/personal_video.h

@@ -4,6 +4,7 @@
 #include <winrt/Windows.Media.Capture.Frames.h>
 
 void PersonalVideo_Initialize();
+void PersonalVideo_Close();
 bool PersonalVideo_SetFormat(uint32_t width, uint32_t height, uint32_t framerate);
 winrt::Windows::Media::Capture::Frames::MediaFrameReader PersonalVideo_CreateFrameReader();
 void PersonalVideo_SetFocus(uint32_t focusmode, uint32_t autofocusrange, uint32_t distance, uint32_t value, uint32_t disabledriverfallback);

hl2ss/hl2ss/stream_pv.cpp

@@ -25,6 +25,7 @@ using namespace winrt::Windows::Perception::Spatial;
 // Global Variables
 //-----------------------------------------------------------------------------
 
+static bool g_ready = false;
 static HANDLE g_quitevent = NULL; // CloseHandle
 static HANDLE g_thread = NULL; // CloseHandle
 
@@ -217,9 +218,13 @@ static void PV_Stream(SOCKET clientsocket)
     ok = recv_u8(clientsocket, mode);
     if (!ok) { return; }
 
+    if (!g_ready && (mode & 4)) { PersonalVideo_Initialize(); g_ready = true; }
+
     ok = ReceiveVideoFormat(clientsocket, format);
     if (!ok) { return; }
 
+    if (!g_ready) { return; }
+
     ok = PersonalVideo_SetFormat(format.width, format.height, format.framerate);
     if (!ok) { return; }
 
@@ -228,7 +233,7 @@ static void PV_Stream(SOCKET clientsocket)
     videoFrameReader = PersonalVideo_CreateFrameReader();
     videoFrameReader.AcquisitionMode(MediaFrameReaderAcquisitionMode::Buffered);
 
-    switch (mode)
+    switch (mode & 3)
     {
     case 0: PV_Stream<false>(clientsocket, clientevent, videoFrameReader, format); break;
     case 1: PV_Stream<true>( clientsocket, clientevent, videoFrameReader, format); break;
@@ -238,6 +243,8 @@ static void PV_Stream(SOCKET clientsocket)
     videoFrameReader.Close();
 
     CloseHandle(clientevent);
+
+    if (g_ready && (mode & 8)) { PersonalVideo_Close(); g_ready = false; }
 }
 
 // OK
@@ -248,8 +255,6 @@ static DWORD WINAPI PV_EntryPoint(void *param)
     SOCKET listensocket; // closesocket
     SOCKET clientsocket; // closesocket
 
-    PersonalVideo_Initialize();
-
     listensocket = CreateSocket(PORT_PV);
 
     ShowMessage("PV: Listening at port %s", PORT_PV);

viewer/client_pv.py

@@ -40,6 +40,9 @@
 
 #------------------------------------------------------------------------------
 
+client = hl2ss.rx_decoded_pv(host, port, hl2ss.ChunkSize.PERSONAL_VIDEO, mode, width, height, framerate, profile, bitrate, 'bgr24')
+client.start_video_subsystem()
+
 if (mode == hl2ss.StreamMode.MODE_2):
     data = hl2ss.download_calibration_pv(host, port, width, height, framerate)
     print('Calibration')
@@ -49,27 +52,27 @@
     print(data.tangential_distortion)
     print(data.projection)
     print(data.intrinsics)
-    quit()
+else:
+    enable = True
 
-enable = True
+    def on_press(key):
+        global enable
+        enable = key != keyboard.Key.esc
+        return enable
 
-def on_press(key):
-    global enable
-    enable = key != keyboard.Key.esc
-    return enable
+    listener = keyboard.Listener(on_press=on_press)
+    listener.start()
 
-listener = keyboard.Listener(on_press=on_press)
-listener.start()
+    client.open()
 
-client = hl2ss.rx_decoded_pv(host, port, hl2ss.ChunkSize.PERSONAL_VIDEO, mode, width, height, framerate, profile, bitrate, 'bgr24')
-client.open()
+    while (enable):
+        data = client.get_next_packet()
+        print('Pose at time {ts}'.format(ts=data.timestamp))
+        print(data.pose)
+        cv2.imshow('Video', data.payload)
+        cv2.waitKey(1)
 
-while (enable):
-    data = client.get_next_packet()
-    print('Pose at time {ts}'.format(ts=data.timestamp))
-    print(data.pose)
-    cv2.imshow('Video', data.payload)
-    cv2.waitKey(1)
+    client.close()
+    listener.join()
 
-client.close()
-listener.join()
+client.stop_video_subsystem()

viewer/client_rm_imu.py

@@ -59,14 +59,14 @@ def on_press(key):
 listener = keyboard.Listener(on_press=on_press)
 listener.start()
 
-client = hl2ss.rx_rm_imu(host, port, chunk_size, mode)
+client = hl2ss.rx_decoded_rm_imu(host, port, chunk_size, mode)
 client.open()
 
 while (enable):
     data = client.get_next_packet()
     print('Pose at time {ts}'.format(ts=data.timestamp))
     print(data.pose)
-    imu_data = hl2ss.unpack_rm_imu(data.payload)
+    imu_data = data.payload
     sample = imu_data.get_frame(0)
     print('Got {count} samples at time {ts}, first sample is (ticks = {st}, x = {x}, y = {y}, z = {z})'.format(count=imu_data.get_count(), ts=data.timestamp, st=sample.sensor_ticks_ns, x=sample.x, y=sample.y, z=sample.z))
 

viewer/client_si.py

@@ -17,11 +17,11 @@
 
 #------------------------------------------------------------------------------
 
-client = hl2ss.rx_si(host, port, hl2ss.ChunkSize.SPATIAL_INPUT)
+client = hl2ss.rx_decoded_si(host, port, hl2ss.ChunkSize.SPATIAL_INPUT)
 client.open()
 
 data = client.get_next_packet()
-si = hl2ss.unpack_si(data.payload)
+si = data.payload
 
 print('Tracking status at time {ts}'.format(ts=data.timestamp))
 

viewer/hl2ss.py

@@ -5,7 +5,6 @@
 import cv2
 import av
 
-_I133_DISABLE_AHAT = True
 
 # Stream TCP Ports
 class StreamPort:
@@ -551,8 +550,6 @@ def _connect_client_rm_vlc(host, port, chunk_size, mode, profile, bitrate):
 
 
 def _connect_client_rm_depth_ahat(host, port, chunk_size, mode, profile, bitrate):
-    if (_I133_DISABLE_AHAT):
-        raise Exception('RM DEPTH AHAT access is currently disabled (https://github.com/microsoft/HoloLens2ForCV/issues/133)')
     c = gatherer()
     c.open(host, port, chunk_size, mode)
     c.sendall(_create_configuration_for_rm_depth_ahat(mode, profile, bitrate))
@@ -691,6 +688,20 @@ def get_next_packet(self):
     def close(self):
         self._client.close()
 
+    def start_video_subsystem(self):
+        mode = self.mode
+        self.mode = 0x7
+        self.open()
+        self.close()
+        self.mode = mode
+
+    def stop_video_subsystem(self):
+        mode = self.mode
+        self.mode = 0xB
+        self.open()
+        self.close()
+        self.mode = mode
+
 
 class rx_microphone:
     def __init__(self, host, port, chunk, profile):
@@ -1037,8 +1048,7 @@ def open(self):
 
     def get_next_packet(self):
         data = self._client.get_next_packet()
-        data.payload = self._codec.decode(data.payload)
-        return data
+        return packet(data.timestamp, self._codec.decode(data.payload), data.pose)
 
     def close(self):
         self._client.close()
@@ -1056,8 +1066,7 @@ def open(self):
 
     def get_next_packet(self):
         data = self._client.get_next_packet()
-        data.payload = self._codec.decode(data.payload)
-        return data
+        return packet(data.timestamp, self._codec.decode(data.payload), data.pose)
 
     def close(self):
         self._client.close()
@@ -1072,8 +1081,22 @@ def open(self):
 
     def get_next_packet(self):
         data = self._client.get_next_packet()
-        data.payload = decode_rm_depth_longthrow(data.payload)
-        return data
+        return packet(data.timestamp, decode_rm_depth_longthrow(data.payload), data.pose)
+
+    def close(self):
+        self._client.close()
+
+
+class rx_decoded_rm_imu:
+    def __init__(self, host, port, chunk, mode):
+        self._client = rx_rm_imu(host, port, chunk, mode)
+
+    def open(self):
+        self._client.open()
+
+    def get_next_packet(self):
+        data = self._client.get_next_packet()
+        return packet(data.timestamp, unpack_rm_imu(data.payload), data.pose)
 
     def close(self):
         self._client.close()
@@ -1092,12 +1115,17 @@ def open(self):
 
     def get_next_packet(self):
         data = self._client.get_next_packet()
-        data.payload = self._codec.decode(data.payload, self._format)
-        return data
+        return packet(data.timestamp, self._codec.decode(data.payload, self._format), data.pose)
 
     def close(self):
         self._client.close()
 
+    def start_video_subsystem(self):
+        self._client.start_video_subsystem()
+
+    def stop_video_subsystem(self):
+        self._client.stop_video_subsystem()
+
 
 class rx_decoded_microphone:
     def __init__(self, host, port, chunk, profile):
@@ -1110,8 +1138,22 @@ def open(self):
 
     def get_next_packet(self):
         data = self._client.get_next_packet()
-        data.payload = self._codec.decode(data.payload)
-        return data
+        return packet(data.timestamp, self._codec.decode(data.payload), data.pose)
+
+    def close(self):
+        self._client.close()
+
+
+class rx_decoded_si:
+    def __init__(self, host, port, chunk):
+        self._client = rx_si(host, port, chunk)
+
+    def open(self):
+        self._client.open()
+
+    def get_next_packet(self):
+        data = self._client.get_next_packet()
+        return packet(data.timestamp, unpack_si(data.payload), data.pose)
 
     def close(self):
         self._client.close()