Skip to content

Commit

Permalink
Support YUV 4:4:4 formats
Browse files Browse the repository at this point in the history
  • Loading branch information
@ns6089
ns6089 committed May 18, 2024
1 parent 3b25154 commit b95eb0a
Show file tree
Hide file tree
Showing 10 changed files with 291 additions and 8 deletions.
7 changes: 6 additions & 1 deletion app/app.pro
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ unix:!macx {
}
win32 {
LIBS += -llibssl -llibcrypto -lSDL2 -lSDL2_ttf -lavcodec -lavutil -lopus -ldxgi -ld3d11
CONFIG += ffmpeg
CONFIG += ffmpeg libplacebo
}
win32:!winrt {
CONFIG += soundio discord-rpc
Expand Down Expand Up @@ -338,6 +338,11 @@ libplacebo {
streaming/video/ffmpeg-renderers/plvk_c.c
HEADERS += \
streaming/video/ffmpeg-renderers/plvk.h

win32 {
INCLUDEPATH += $((VULKAN_SDK))/Include
LIBS += libplacebo.lib
}
}
config_EGL {
message(EGL renderer selected)
Expand Down
12 changes: 12 additions & 0 deletions app/gui/SettingsView.qml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1611,6 +1611,18 @@ Flickable {
qsTr("HDR streaming is not supported on this PC.")
}

CheckBox {
id: enableYUV444
width: parent.width
text: qsTr("Enable YUV 4:4:4 (Experimental)")
font.pointSize: 12

checked: StreamingPreferences.enableYUV444
onCheckedChanged: {
StreamingPreferences.enableYUV444 = checked
}
}

CheckBox {
id: enableMdns
width: parent.width
Expand Down
3 changes: 3 additions & 0 deletions app/settings/streamingpreferences.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
#define SER_AUDIOCFG "audiocfg"
#define SER_VIDEOCFG "videocfg"
#define SER_HDR "hdr"
#define SER_YUV444 "yuv444"
#define SER_VIDEODEC "videodec"
#define SER_WINDOWMODE "windowmode"
#define SER_MDNS "mdns"
Expand Down Expand Up @@ -140,6 +141,7 @@ void StreamingPreferences::reload()
swapFaceButtons = settings.value(SER_SWAPFACEBUTTONS, false).toBool();
keepAwake = settings.value(SER_KEEPAWAKE, true).toBool();
enableHdr = settings.value(SER_HDR, false).toBool();
enableYUV444 = settings.value(SER_YUV444, false).toBool();
captureSysKeysMode = static_cast<CaptureSysKeysMode>(settings.value(SER_CAPTURESYSKEYS,
static_cast<int>(CaptureSysKeysMode::CSK_OFF)).toInt());
audioConfig = static_cast<AudioConfig>(settings.value(SER_AUDIOCFG,
Expand Down Expand Up @@ -320,6 +322,7 @@ void StreamingPreferences::save()
settings.setValue(SER_SHOWPERFOVERLAY, showPerformanceOverlay);
settings.setValue(SER_AUDIOCFG, static_cast<int>(audioConfig));
settings.setValue(SER_HDR, enableHdr);
settings.setValue(SER_YUV444, enableYUV444);
settings.setValue(SER_VIDEOCFG, static_cast<int>(videoCodecConfig));
settings.setValue(SER_VIDEODEC, static_cast<int>(videoDecoderSelection));
settings.setValue(SER_WINDOWMODE, static_cast<int>(windowMode));
Expand Down
3 changes: 3 additions & 0 deletions app/settings/streamingpreferences.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@ class StreamingPreferences : public QObject
Q_PROPERTY(AudioConfig audioConfig MEMBER audioConfig NOTIFY audioConfigChanged)
Q_PROPERTY(VideoCodecConfig videoCodecConfig MEMBER videoCodecConfig NOTIFY videoCodecConfigChanged)
Q_PROPERTY(bool enableHdr MEMBER enableHdr NOTIFY enableHdrChanged)
Q_PROPERTY(bool enableYUV444 MEMBER enableYUV444 NOTIFY enableYUV444Changed)
Q_PROPERTY(VideoDecoderSelection videoDecoderSelection MEMBER videoDecoderSelection NOTIFY videoDecoderSelectionChanged)
Q_PROPERTY(WindowMode windowMode MEMBER windowMode NOTIFY windowModeChanged)
Q_PROPERTY(WindowMode recommendedFullScreenMode MEMBER recommendedFullScreenMode CONSTANT)
Expand Down Expand Up @@ -168,6 +169,7 @@ class StreamingPreferences : public QObject
AudioConfig audioConfig;
VideoCodecConfig videoCodecConfig;
bool enableHdr;
bool enableYUV444;
VideoDecoderSelection videoDecoderSelection;
WindowMode windowMode;
WindowMode recommendedFullScreenMode;
Expand All @@ -190,6 +192,7 @@ class StreamingPreferences : public QObject
void audioConfigChanged();
void videoCodecConfigChanged();
void enableHdrChanged();
void enableYUV444Changed();
void videoDecoderSelectionChanged();
void uiDisplayModeChanged();
void windowModeChanged();
Expand Down
59 changes: 57 additions & 2 deletions app/streaming/session.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -472,18 +472,33 @@ bool Session::populateDecoderProperties(SDL_Window* window)
IVideoDecoder* decoder;

int videoFormat;
if (m_StreamConfig.supportedVideoFormats & VIDEO_FORMAT_AV1_MAIN10) {
if (m_StreamConfig.supportedVideoFormats & VIDEO_FORMAT_AV1_HIGH10_444) {
videoFormat = VIDEO_FORMAT_AV1_HIGH10_444;
}
else if (m_StreamConfig.supportedVideoFormats & VIDEO_FORMAT_AV1_MAIN10) {
videoFormat = VIDEO_FORMAT_AV1_MAIN10;
}
else if (m_StreamConfig.supportedVideoFormats & VIDEO_FORMAT_AV1_HIGH8_444) {
videoFormat = VIDEO_FORMAT_AV1_HIGH8_444;
}
else if (m_StreamConfig.supportedVideoFormats & VIDEO_FORMAT_AV1_MAIN8) {
videoFormat = VIDEO_FORMAT_AV1_MAIN8;
}
else if (m_StreamConfig.supportedVideoFormats & VIDEO_FORMAT_H265_REXT10_444) {
videoFormat = VIDEO_FORMAT_H265_REXT10_444;
}
else if (m_StreamConfig.supportedVideoFormats & VIDEO_FORMAT_H265_MAIN10) {
videoFormat = VIDEO_FORMAT_H265_MAIN10;
}
else if (m_StreamConfig.supportedVideoFormats & VIDEO_FORMAT_H265_REXT8_444) {
videoFormat = VIDEO_FORMAT_H265_REXT8_444;
}
else if (m_StreamConfig.supportedVideoFormats & VIDEO_FORMAT_H265) {
videoFormat = VIDEO_FORMAT_H265;
}
else if (m_StreamConfig.supportedVideoFormats & VIDEO_FORMAT_H264_HIGH8_444) {
videoFormat = VIDEO_FORMAT_H264_HIGH8_444;
}
else {
videoFormat = VIDEO_FORMAT_H264;
}
Expand Down Expand Up @@ -712,14 +727,31 @@ bool Session::initialize()
#endif

// TODO: Determine if HEVC is better depending on the decoder
if (m_Preferences->enableHdr && isHardwareDecodeAvailable(testWindow,
if (m_Preferences->enableHdr &&
m_Preferences->enableYUV444 && isHardwareDecodeAvailable(testWindow,
m_Preferences->videoDecoderSelection,
VIDEO_FORMAT_H265_REXT10_444,
m_StreamConfig.width,
m_StreamConfig.height,
m_StreamConfig.fps)) {
m_StreamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265 | VIDEO_FORMAT_H265_MAIN10 | VIDEO_FORMAT_H265_REXT8_444 | VIDEO_FORMAT_H265_REXT10_444;
}
else if (m_Preferences->enableHdr && isHardwareDecodeAvailable(testWindow,
m_Preferences->videoDecoderSelection,
VIDEO_FORMAT_H265_MAIN10,
m_StreamConfig.width,
m_StreamConfig.height,
m_StreamConfig.fps)) {
m_StreamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265 | VIDEO_FORMAT_H265_MAIN10;
}
else if (m_Preferences->enableYUV444 && isHardwareDecodeAvailable(testWindow,
m_Preferences->videoDecoderSelection,
VIDEO_FORMAT_H265_REXT8_444,
m_StreamConfig.width,
m_StreamConfig.height,
m_StreamConfig.fps)) {
m_StreamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265 | VIDEO_FORMAT_H265_REXT8_444;
}
else if (isHardwareDecodeAvailable(testWindow,
m_Preferences->videoDecoderSelection,
VIDEO_FORMAT_H265,
Expand Down Expand Up @@ -747,18 +779,33 @@ bool Session::initialize()
#endif
break;
case StreamingPreferences::VCC_FORCE_H264:
if (m_Preferences->enableYUV444) {
m_StreamConfig.supportedVideoFormats |= VIDEO_FORMAT_H264_HIGH8_444;
}
break;
case StreamingPreferences::VCC_FORCE_HEVC:
case StreamingPreferences::VCC_FORCE_HEVC_HDR_DEPRECATED:
m_StreamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265;
if (m_Preferences->enableHdr) {
m_StreamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265_MAIN10;
if (m_Preferences->enableYUV444) {
m_StreamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265_REXT10_444;
}
}
if (m_Preferences->enableYUV444) {
m_StreamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265_REXT8_444;
}
break;
case StreamingPreferences::VCC_FORCE_AV1:
m_StreamConfig.supportedVideoFormats |= VIDEO_FORMAT_AV1_MAIN8;
if (m_Preferences->enableHdr) {
m_StreamConfig.supportedVideoFormats |= VIDEO_FORMAT_AV1_MAIN10;
if (m_Preferences->enableYUV444) {
m_StreamConfig.supportedVideoFormats |= VIDEO_FORMAT_AV1_HIGH10_444;
}
}
if (m_Preferences->enableYUV444) {
m_StreamConfig.supportedVideoFormats |= VIDEO_FORMAT_AV1_HIGH8_444;
}

// We'll try to fall back to HEVC first if AV1 fails. We'd rather not fall back
Expand All @@ -769,6 +816,12 @@ bool Session::initialize()
if (m_StreamConfig.supportedVideoFormats & VIDEO_FORMAT_AV1_MAIN10) {
m_StreamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265_MAIN10;
}
if (m_StreamConfig.supportedVideoFormats & VIDEO_FORMAT_AV1_HIGH8_444) {
m_StreamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265_REXT8_444;
}
if (m_StreamConfig.supportedVideoFormats & VIDEO_FORMAT_AV1_HIGH10_444) {
m_StreamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265_REXT10_444;
}
break;
}

Expand Down Expand Up @@ -988,6 +1041,8 @@ bool Session::validateLaunch(SDL_Window* testWindow)
}
}

// TODO: decide what to do with YUV444 here

if (m_StreamConfig.width >= 3840) {
// Only allow 4K on GFE 3.x+
if (m_Computer->gfeVersion.isEmpty() || m_Computer->gfeVersion.startsWith("2.")) {
Expand Down
15 changes: 12 additions & 3 deletions app/streaming/video/ffmpeg-renderers/plvk.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,8 @@ bool PlVkRenderer::tryInitializeDevice(VkPhysicalDevice device, VkPhysicalDevice
return false;
}

if ((decoderParams->videoFormat & VIDEO_FORMAT_MASK_10BIT) && !isColorSpaceSupportedByPhysicalDevice(device, VK_COLOR_SPACE_HDR10_ST2084_EXT)) {
// TODO: support 10-bit SDR without this hack
if (false && (decoderParams->videoFormat & VIDEO_FORMAT_MASK_10BIT) && !isColorSpaceSupportedByPhysicalDevice(device, VK_COLOR_SPACE_HDR10_ST2084_EXT)) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"Vulkan device '%s' does not support HDR10 (ST.2084 PQ)",
deviceProps->deviceName);
Expand Down Expand Up @@ -898,17 +899,25 @@ int PlVkRenderer::getRendererAttributes()
{
int attributes = 0;

if (isColorSpaceSupportedByPhysicalDevice(m_Vulkan->phys_device, VK_COLOR_SPACE_HDR10_ST2084_EXT)) {
// TODO: support 10-bit SDR without this hack
if (true || isColorSpaceSupportedByPhysicalDevice(m_Vulkan->phys_device, VK_COLOR_SPACE_HDR10_ST2084_EXT)) {
attributes |= RENDERER_ATTRIBUTE_HDR_SUPPORT;
}

return attributes;
}

int PlVkRenderer::getDecoderColorspace()
{
// TODO: for testing color inaccuracies
return COLORSPACE_REC_709;
}

int PlVkRenderer::getDecoderColorRange()
{
// Explicitly set the color range to full to fix raised black levels on OLED displays
return COLOR_RANGE_FULL;
// TODO: nvenc seems to produce wrong colors in full range, decide what to do here
return COLOR_RANGE_LIMITED;
}

int PlVkRenderer::getDecoderCapabilities()
Expand Down
1 change: 1 addition & 0 deletions app/streaming/video/ffmpeg-renderers/plvk.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ class PlVkRenderer : public IFFmpegRenderer {
virtual void notifyOverlayUpdated(Overlay::OverlayType) override;
virtual bool notifyWindowChanged(PWINDOW_STATE_CHANGE_INFO) override;
virtual int getRendererAttributes() override;
virtual int getDecoderColorspace() override;
virtual int getDecoderColorRange() override;
virtual int getDecoderCapabilities() override;
virtual bool needsTestFrame() override;
Expand Down
66 changes: 64 additions & 2 deletions app/streaming/video/ffmpeg.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -525,6 +525,26 @@ bool FFmpegVideoDecoder::completeInitialization(const AVCodec* decoder, enum AVP
m_Pkt->data = (uint8_t*)k_AV1Main10TestFrame;
m_Pkt->size = sizeof(k_AV1Main10TestFrame);
break;
case VIDEO_FORMAT_H264_HIGH8_444:
m_Pkt->data = (uint8_t*)k_h264High_444TestFrame;
m_Pkt->size = sizeof(k_h264High_444TestFrame);
break;
case VIDEO_FORMAT_H265_REXT8_444:
m_Pkt->data = (uint8_t*)k_HEVCRExt8_444TestFrame;
m_Pkt->size = sizeof(k_HEVCRExt8_444TestFrame);
break;
case VIDEO_FORMAT_H265_REXT10_444:
m_Pkt->data = (uint8_t*)k_HEVCRExt10_444TestFrame;
m_Pkt->size = sizeof(k_HEVCRExt10_444TestFrame);
break;
case VIDEO_FORMAT_AV1_HIGH8_444:
m_Pkt->data = (uint8_t*)k_AV1High8_444TestFrame;
m_Pkt->size = sizeof(k_AV1High8_444TestFrame);
break;
case VIDEO_FORMAT_AV1_HIGH10_444:
m_Pkt->data = (uint8_t*)k_AV1High10_444TestFrame;
m_Pkt->size = sizeof(k_AV1High10_444TestFrame);
break;
default:
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"No test frame for format: %x",
Expand Down Expand Up @@ -677,23 +697,44 @@ void FFmpegVideoDecoder::stringifyVideoStats(VIDEO_STATS& stats, char* output, i
codecString = "H.264";
break;

case VIDEO_FORMAT_H264_HIGH8_444:
codecString = "H.264 4:4:4";
break;

case VIDEO_FORMAT_H265:
codecString = "HEVC";
break;

case VIDEO_FORMAT_H265_REXT8_444:
codecString = "HEVC 4:4:4";
break;

case VIDEO_FORMAT_H265_MAIN10:
if (LiGetCurrentHostDisplayHdrMode()) {
codecString = "HEVC Main 10 HDR";
codecString = "HEVC 10-bit HDR";
}
else {
codecString = "HEVC Main 10 SDR";
codecString = "HEVC 10-bit SDR";
}
break;

case VIDEO_FORMAT_H265_REXT10_444:
if (LiGetCurrentHostDisplayHdrMode()) {
codecString = "HEVC 10-bit HDR 4:4:4";
}
else {
codecString = "HEVC 10-bit SDR 4:4:4";
}
break;

case VIDEO_FORMAT_AV1_MAIN8:
codecString = "AV1";
break;

case VIDEO_FORMAT_AV1_HIGH8_444:
codecString = "AV1 4:4:4";
break;

case VIDEO_FORMAT_AV1_MAIN10:
if (LiGetCurrentHostDisplayHdrMode()) {
codecString = "AV1 10-bit HDR";
Expand All @@ -703,6 +744,15 @@ void FFmpegVideoDecoder::stringifyVideoStats(VIDEO_STATS& stats, char* output, i
}
break;

case VIDEO_FORMAT_AV1_HIGH10_444:
if (LiGetCurrentHostDisplayHdrMode()) {
codecString = "AV1 10-bit HDR 4:4:4";
}
else {
codecString = "AV1 10-bit SDR 4:4:4";
}
break;

default:
SDL_assert(false);
codecString = "UNKNOWN";
Expand Down Expand Up @@ -1238,6 +1288,12 @@ bool FFmpegVideoDecoder::initialize(PDECODER_PARAMETERS params)
break;
}

// TODO: reexamine this
if ((params->videoFormat & VIDEO_FORMAT_MASK_YUV444) && config->device_type != AV_HWDEVICE_TYPE_VULKAN) {
// We only support YUV 4:4:4 decoding on Vulkan through libplacebo
continue;
}

// Initialize the hardware codec and submit a test frame if the renderer needs it
IFFmpegRenderer::InitFailureReason failureReason;
if (tryInitializeRenderer(decoder, AV_PIX_FMT_NONE, params, config, &failureReason,
Expand Down Expand Up @@ -1323,6 +1379,12 @@ bool FFmpegVideoDecoder::initialize(PDECODER_PARAMETERS params)
break;
}

// TODO: reexamine this
if ((params->videoFormat & VIDEO_FORMAT_MASK_YUV444) && config->device_type != AV_HWDEVICE_TYPE_VULKAN) {
// We only support YUV 4:4:4 decoding on Vulkan through libplacebo
continue;
}

// Initialize the hardware codec and submit a test frame if the renderer needs it
IFFmpegRenderer::InitFailureReason failureReason;
if (tryInitializeRenderer(decoder, AV_PIX_FMT_NONE, params, config, &failureReason,
Expand Down
6 changes: 6 additions & 0 deletions app/streaming/video/ffmpeg.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,4 +104,10 @@ class FFmpegVideoDecoder : public IVideoDecoder {
static const uint8_t k_HEVCMain10TestFrame[];
static const uint8_t k_AV1Main8TestFrame[];
static const uint8_t k_AV1Main10TestFrame[];
static const uint8_t k_h264High_444TestFrame[];
static const uint8_t k_HEVCRExt8_444TestFrame[];
static const uint8_t k_HEVCRExt10_444TestFrame[];
static const uint8_t k_AV1High8_444TestFrame[];
static const uint8_t k_AV1High10_444TestFrame[];

};
Loading

0 comments on commit b95eb0a

Please sign in to comment.