From 33b8339568d5f5ae34980a7b1e3861b530c7e0b2 Mon Sep 17 00:00:00 2001 From: jdibenes <58836755+jdibenes@users.noreply.github.com> Date: Mon, 14 Nov 2022 17:26:03 -0500 Subject: [PATCH] pv / rm depth ahat coop model https://github.com/microsoft/HoloLens2ForCV/issues/133 --- hl2ss/hl2ss/personal_video.cpp | 8 ++++ hl2ss/hl2ss/personal_video.h | 1 + hl2ss/hl2ss/stream_pv.cpp | 11 ++++-- viewer/client_pv.py | 39 ++++++++++--------- viewer/client_rm_imu.py | 4 +- viewer/client_si.py | 4 +- viewer/hl2ss.py | 68 +++++++++++++++++++++++++++------- 7 files changed, 97 insertions(+), 38 deletions(-) diff --git a/hl2ss/hl2ss/personal_video.cpp b/hl2ss/hl2ss/personal_video.cpp index d41b7ec91..baea8ef38 100644 --- a/hl2ss/hl2ss/personal_video.cpp +++ b/hl2ss/hl2ss/personal_video.cpp @@ -105,6 +105,14 @@ void PersonalVideo_Initialize() PersonalVideo_FindVideoSource(g_mediaCapture, g_videoSource); } +// OK +void PersonalVideo_Close() +{ + g_videoSource = nullptr; + g_mediaCapture.Close(); + g_mediaCapture = nullptr; +} + // OK bool PersonalVideo_SetFormat(uint32_t width, uint32_t height, uint32_t framerate) { diff --git a/hl2ss/hl2ss/personal_video.h b/hl2ss/hl2ss/personal_video.h index 4976eaad6..581741d92 100644 --- a/hl2ss/hl2ss/personal_video.h +++ b/hl2ss/hl2ss/personal_video.h @@ -4,6 +4,7 @@ #include void PersonalVideo_Initialize(); +void PersonalVideo_Close(); bool PersonalVideo_SetFormat(uint32_t width, uint32_t height, uint32_t framerate); winrt::Windows::Media::Capture::Frames::MediaFrameReader PersonalVideo_CreateFrameReader(); void PersonalVideo_SetFocus(uint32_t focusmode, uint32_t autofocusrange, uint32_t distance, uint32_t value, uint32_t disabledriverfallback); diff --git a/hl2ss/hl2ss/stream_pv.cpp b/hl2ss/hl2ss/stream_pv.cpp index c5d9c5129..6754e872c 100644 --- a/hl2ss/hl2ss/stream_pv.cpp +++ b/hl2ss/hl2ss/stream_pv.cpp @@ -25,6 +25,7 @@ using namespace winrt::Windows::Perception::Spatial; // Global Variables //----------------------------------------------------------------------------- +static bool g_ready = false; static HANDLE g_quitevent = NULL; // CloseHandle static HANDLE g_thread = NULL; // CloseHandle @@ -217,9 +218,13 @@ static void PV_Stream(SOCKET clientsocket) ok = recv_u8(clientsocket, mode); if (!ok) { return; } + if (!g_ready && (mode & 4)) { PersonalVideo_Initialize(); g_ready = true; } + ok = ReceiveVideoFormat(clientsocket, format); if (!ok) { return; } + if (!g_ready) { return; } + ok = PersonalVideo_SetFormat(format.width, format.height, format.framerate); if (!ok) { return; } @@ -228,7 +233,7 @@ static void PV_Stream(SOCKET clientsocket) videoFrameReader = PersonalVideo_CreateFrameReader(); videoFrameReader.AcquisitionMode(MediaFrameReaderAcquisitionMode::Buffered); - switch (mode) + switch (mode & 3) { case 0: PV_Stream(clientsocket, clientevent, videoFrameReader, format); break; case 1: PV_Stream( clientsocket, clientevent, videoFrameReader, format); break; @@ -238,6 +243,8 @@ static void PV_Stream(SOCKET clientsocket) videoFrameReader.Close(); CloseHandle(clientevent); + + if (g_ready && (mode & 8)) { PersonalVideo_Close(); g_ready = false; } } // OK @@ -248,8 +255,6 @@ static DWORD WINAPI PV_EntryPoint(void *param) SOCKET listensocket; // closesocket SOCKET clientsocket; // closesocket - PersonalVideo_Initialize(); - listensocket = CreateSocket(PORT_PV); ShowMessage("PV: Listening at port %s", PORT_PV); diff --git a/viewer/client_pv.py b/viewer/client_pv.py index b72f70580..28a1259b2 100644 --- a/viewer/client_pv.py +++ b/viewer/client_pv.py @@ -40,6 +40,9 @@ #------------------------------------------------------------------------------ +client = hl2ss.rx_decoded_pv(host, port, hl2ss.ChunkSize.PERSONAL_VIDEO, mode, width, height, framerate, profile, bitrate, 'bgr24') +client.start_video_subsystem() + if (mode == hl2ss.StreamMode.MODE_2): data = hl2ss.download_calibration_pv(host, port, width, height, framerate) print('Calibration') @@ -49,27 +52,27 @@ print(data.tangential_distortion) print(data.projection) print(data.intrinsics) - quit() +else: + enable = True -enable = True + def on_press(key): + global enable + enable = key != keyboard.Key.esc + return enable -def on_press(key): - global enable - enable = key != keyboard.Key.esc - return enable + listener = keyboard.Listener(on_press=on_press) + listener.start() -listener = keyboard.Listener(on_press=on_press) -listener.start() + client.open() -client = hl2ss.rx_decoded_pv(host, port, hl2ss.ChunkSize.PERSONAL_VIDEO, mode, width, height, framerate, profile, bitrate, 'bgr24') -client.open() + while (enable): + data = client.get_next_packet() + print('Pose at time {ts}'.format(ts=data.timestamp)) + print(data.pose) + cv2.imshow('Video', data.payload) + cv2.waitKey(1) -while (enable): - data = client.get_next_packet() - print('Pose at time {ts}'.format(ts=data.timestamp)) - print(data.pose) - cv2.imshow('Video', data.payload) - cv2.waitKey(1) + client.close() + listener.join() -client.close() -listener.join() +client.stop_video_subsystem() diff --git a/viewer/client_rm_imu.py b/viewer/client_rm_imu.py index 7d189c73c..4d6a00e97 100644 --- a/viewer/client_rm_imu.py +++ b/viewer/client_rm_imu.py @@ -59,14 +59,14 @@ def on_press(key): listener = keyboard.Listener(on_press=on_press) listener.start() -client = hl2ss.rx_rm_imu(host, port, chunk_size, mode) +client = hl2ss.rx_decoded_rm_imu(host, port, chunk_size, mode) client.open() while (enable): data = client.get_next_packet() print('Pose at time {ts}'.format(ts=data.timestamp)) print(data.pose) - imu_data = hl2ss.unpack_rm_imu(data.payload) + imu_data = data.payload sample = imu_data.get_frame(0) print('Got {count} samples at time {ts}, first sample is (ticks = {st}, x = {x}, y = {y}, z = {z})'.format(count=imu_data.get_count(), ts=data.timestamp, st=sample.sensor_ticks_ns, x=sample.x, y=sample.y, z=sample.z)) diff --git a/viewer/client_si.py b/viewer/client_si.py index 30b4f117e..a171040ba 100644 --- a/viewer/client_si.py +++ b/viewer/client_si.py @@ -17,11 +17,11 @@ #------------------------------------------------------------------------------ -client = hl2ss.rx_si(host, port, hl2ss.ChunkSize.SPATIAL_INPUT) +client = hl2ss.rx_decoded_si(host, port, hl2ss.ChunkSize.SPATIAL_INPUT) client.open() data = client.get_next_packet() -si = hl2ss.unpack_si(data.payload) +si = data.payload print('Tracking status at time {ts}'.format(ts=data.timestamp)) diff --git a/viewer/hl2ss.py b/viewer/hl2ss.py index f4475f6fe..f837256e2 100644 --- a/viewer/hl2ss.py +++ b/viewer/hl2ss.py @@ -5,7 +5,6 @@ import cv2 import av -_I133_DISABLE_AHAT = True # Stream TCP Ports class StreamPort: @@ -551,8 +550,6 @@ def _connect_client_rm_vlc(host, port, chunk_size, mode, profile, bitrate): def _connect_client_rm_depth_ahat(host, port, chunk_size, mode, profile, bitrate): - if (_I133_DISABLE_AHAT): - raise Exception('RM DEPTH AHAT access is currently disabled (https://github.com/microsoft/HoloLens2ForCV/issues/133)') c = gatherer() c.open(host, port, chunk_size, mode) c.sendall(_create_configuration_for_rm_depth_ahat(mode, profile, bitrate)) @@ -691,6 +688,20 @@ def get_next_packet(self): def close(self): self._client.close() + def start_video_subsystem(self): + mode = self.mode + self.mode = 0x7 + self.open() + self.close() + self.mode = mode + + def stop_video_subsystem(self): + mode = self.mode + self.mode = 0xB + self.open() + self.close() + self.mode = mode + class rx_microphone: def __init__(self, host, port, chunk, profile): @@ -1037,8 +1048,7 @@ def open(self): def get_next_packet(self): data = self._client.get_next_packet() - data.payload = self._codec.decode(data.payload) - return data + return packet(data.timestamp, self._codec.decode(data.payload), data.pose) def close(self): self._client.close() @@ -1056,8 +1066,7 @@ def open(self): def get_next_packet(self): data = self._client.get_next_packet() - data.payload = self._codec.decode(data.payload) - return data + return packet(data.timestamp, self._codec.decode(data.payload), data.pose) def close(self): self._client.close() @@ -1072,8 +1081,22 @@ def open(self): def get_next_packet(self): data = self._client.get_next_packet() - data.payload = decode_rm_depth_longthrow(data.payload) - return data + return packet(data.timestamp, decode_rm_depth_longthrow(data.payload), data.pose) + + def close(self): + self._client.close() + + +class rx_decoded_rm_imu: + def __init__(self, host, port, chunk, mode): + self._client = rx_rm_imu(host, port, chunk, mode) + + def open(self): + self._client.open() + + def get_next_packet(self): + data = self._client.get_next_packet() + return packet(data.timestamp, unpack_rm_imu(data.payload), data.pose) def close(self): self._client.close() @@ -1092,12 +1115,17 @@ def open(self): def get_next_packet(self): data = self._client.get_next_packet() - data.payload = self._codec.decode(data.payload, self._format) - return data + return packet(data.timestamp, self._codec.decode(data.payload, self._format), data.pose) def close(self): self._client.close() + def start_video_subsystem(self): + self._client.start_video_subsystem() + + def stop_video_subsystem(self): + self._client.stop_video_subsystem() + class rx_decoded_microphone: def __init__(self, host, port, chunk, profile): @@ -1110,8 +1138,22 @@ def open(self): def get_next_packet(self): data = self._client.get_next_packet() - data.payload = self._codec.decode(data.payload) - return data + return packet(data.timestamp, self._codec.decode(data.payload), data.pose) + + def close(self): + self._client.close() + + +class rx_decoded_si: + def __init__(self, host, port, chunk): + self._client = rx_si(host, port, chunk) + + def open(self): + self._client.open() + + def get_next_packet(self): + data = self._client.get_next_packet() + return packet(data.timestamp, unpack_si(data.payload), data.pose) def close(self): self._client.close()