Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(win/video): add support for recombined YUV444 encoding #2760

Closed
wants to merge 1 commit into from
Closed

feat(win/video): add support for recombined YUV444 encoding #2760

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 13 additions & 4 deletions src/nvenc/nvenc_base.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,16 @@ namespace nvenc {
if (encoder) destroy_encoder();
auto fail_guard = util::fail_guard([this] { destroy_encoder(); });

encoder_params.width = client_config.width;
encoder_params.height = client_config.height;
if (client_config.chromaSamplingType == 2) {
// YUV 4:4:4 recombined into YUV 4:2:0
auto recombined_dimensions = video::calculate_yuv444in420_dimensions(client_config.width, client_config.height);
encoder_params.width = recombined_dimensions.width;
encoder_params.height = recombined_dimensions.height;
}
else {
encoder_params.width = client_config.width;
encoder_params.height = client_config.height;
}
encoder_params.buffer_format = buffer_format;
encoder_params.rfi = true;

Expand Down Expand Up @@ -288,7 +296,7 @@ namespace nvenc {
vui_config.colourPrimaries = colorspace.primaries;
vui_config.transferCharacteristics = colorspace.tranfer_function;
vui_config.colourMatrix = colorspace.matrix;
vui_config.chromaSampleLocationFlag = buffer_is_yuv444() ? 0 : 1;
vui_config.chromaSampleLocationFlag = (client_config.chromaSamplingType == 0) ? 1 : 0;
vui_config.chromaSampleLocationTop = 0;
vui_config.chromaSampleLocationBot = 0;
};
Expand Down Expand Up @@ -341,7 +349,7 @@ namespace nvenc {
format_config.transferCharacteristics = colorspace.tranfer_function;
format_config.matrixCoefficients = colorspace.matrix;
format_config.colorRange = colorspace.full_range;
format_config.chromaSamplePosition = buffer_is_yuv444() ? 0 : 1;
format_config.chromaSamplePosition = (client_config.chromaSamplingType == 0) ? 1 : 0;
set_ref_frames(format_config.maxNumRefFramesInDPB, format_config.numFwdRefs, 8);
set_minqp_if_enabled(config.min_qp_av1);

Expand Down Expand Up @@ -395,6 +403,7 @@ namespace nvenc {
std::string extra;
if (init_params.enableEncodeAsync) extra += " async";
if (buffer_is_yuv444()) extra += " yuv444";
if (client_config.chromaSamplingType == 2) extra += " yuv444in420";
if (buffer_is_10bit()) extra += " 10-bit";
if (enc_config.rcParams.multiPass != NV_ENC_MULTI_PASS_DISABLED) extra += " two-pass";
if (config.vbv_percentage_increase > 0 && get_encoder_cap(NV_ENC_CAPS_SUPPORT_CUSTOM_VBV_BUF_SIZE)) extra += " vbv+" + std::to_string(config.vbv_percentage_increase);
Expand Down
15 changes: 15 additions & 0 deletions src/nvhttp.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -722,30 +722,45 @@ namespace nvhttp {
uint32_t codec_mode_flags = SCM_H264;
if (video::last_encoder_probe_supported_yuv444_for_codec[0]) {
codec_mode_flags |= SCM_H264_HIGH8_444;
if (video::last_encoder_probe_supported_yuv444in420) {
codec_mode_flags |= SCM_H264_HIGH8_444IN420;
}
}
if (video::active_hevc_mode >= 2) {
codec_mode_flags |= SCM_HEVC;
if (video::last_encoder_probe_supported_yuv444_for_codec[1]) {
codec_mode_flags |= SCM_HEVC_REXT8_444;
}
if (video::last_encoder_probe_supported_yuv444in420) {
codec_mode_flags |= SCM_HEVC_MAIN8_444IN420;
}
}
if (video::active_hevc_mode >= 3) {
codec_mode_flags |= SCM_HEVC_MAIN10;
if (video::last_encoder_probe_supported_yuv444_for_codec[1]) {
codec_mode_flags |= SCM_HEVC_REXT10_444;
}
if (video::last_encoder_probe_supported_yuv444in420) {
codec_mode_flags |= SCM_HEVC_MAIN10_444IN420;
}
}
if (video::active_av1_mode >= 2) {
codec_mode_flags |= SCM_AV1_MAIN8;
if (video::last_encoder_probe_supported_yuv444_for_codec[2]) {
codec_mode_flags |= SCM_AV1_HIGH8_444;
}
if (video::last_encoder_probe_supported_yuv444in420) {
codec_mode_flags |= SCM_AV1_MAIN8_444IN420;
}
}
if (video::active_av1_mode >= 3) {
codec_mode_flags |= SCM_AV1_MAIN10;
if (video::last_encoder_probe_supported_yuv444_for_codec[2]) {
codec_mode_flags |= SCM_AV1_HIGH10_444;
}
if (video::last_encoder_probe_supported_yuv444in420) {
codec_mode_flags |= SCM_AV1_MAIN10_444IN420;
}
}
tree.put("root.ServerCodecModeSupport", codec_mode_flags);

Expand Down
25 changes: 23 additions & 2 deletions src/platform/common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -483,13 +483,25 @@ namespace platf {
virtual int
dummy_img(img_t *img) = 0;

/**
* @brief Create AVCodec encode device.
* @param pix_fmt_e Surface format of the encoder.
* @param yuv444in420 Whether YUV 4:4:4 recombination into YUV 4:2:0 must be performed.
* @return `unique_ptr` with `avcodec_encode_device_t` implementation on success, `nullptr` on failure.
*/
virtual std::unique_ptr<avcodec_encode_device_t>
make_avcodec_encode_device(pix_fmt_e pix_fmt) {
make_avcodec_encode_device(pix_fmt_e pix_fmt, bool yuv444in420) {
return nullptr;
}

/**
* @brief Create NVENC encode device.
* @param pix_fmt_e Surface format of the encoder.
* @param yuv444in420 Whether YUV 4:4:4 recombination into YUV 4:2:0 must be performed.
* @return `unique_ptr` with `nvenc_encode_device_t` implementation on success, `nullptr` on failure.
*/
virtual std::unique_ptr<nvenc_encode_device_t>
make_nvenc_encode_device(pix_fmt_e pix_fmt) {
make_nvenc_encode_device(pix_fmt_e pix_fmt, bool yuv444in420) {
return nullptr;
}

Expand All @@ -515,6 +527,15 @@ namespace platf {
return true;
}

/**
* @brief Check if YUV 4:4:4 recombination into YUV 4:2:0 is supported by the display device.
* @return `true` if supported, `false` otherwise.
*/
virtual bool
is_yuv444in420_supported() {
return false;
}

virtual ~display_t() = default;

// Offsets for when streaming a specific monitor. By default, they are 0.
Expand Down
7 changes: 6 additions & 1 deletion src/platform/linux/cuda.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -961,7 +961,12 @@ namespace cuda {
}

std::unique_ptr<platf::avcodec_encode_device_t>
make_avcodec_encode_device(platf::pix_fmt_e pix_fmt) {
make_avcodec_encode_device(platf::pix_fmt_e pix_fmt, bool yuv444in420) {
if (yuv444in420) {
BOOST_LOG(error) << "Recombined YUV 4:4:4 is not supported";
return nullptr;
}

return ::cuda::make_avcodec_encode_device(width, height, true);
}

Expand Down
14 changes: 12 additions & 2 deletions src/platform/linux/kmsgrab.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1236,7 +1236,12 @@ namespace platf {
}

std::unique_ptr<avcodec_encode_device_t>
make_avcodec_encode_device(pix_fmt_e pix_fmt) override {
make_avcodec_encode_device(pix_fmt_e pix_fmt, bool yuv444in420) override {
if (yuv444in420) {
BOOST_LOG(error) << "Recombined YUV 4:4:4 is not supported";
return nullptr;
}

#ifdef SUNSHINE_BUILD_VAAPI
if (mem_type == mem_type_e::vaapi) {
return va::make_avcodec_encode_device(width, height, false);
Expand Down Expand Up @@ -1373,7 +1378,12 @@ namespace platf {
display_t(mem_type) {}

std::unique_ptr<avcodec_encode_device_t>
make_avcodec_encode_device(pix_fmt_e pix_fmt) override {
make_avcodec_encode_device(pix_fmt_e pix_fmt, bool yuv444in420) override {
if (yuv444in420) {
BOOST_LOG(error) << "Recombined YUV 4:4:4 is not supported";
return nullptr;
}

#ifdef SUNSHINE_BUILD_VAAPI
if (mem_type == mem_type_e::vaapi) {
return va::make_avcodec_encode_device(width, height, dup(card.render_fd.el), img_offset_x, img_offset_y, true);
Expand Down
14 changes: 12 additions & 2 deletions src/platform/linux/wlgrab.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,12 @@ namespace wl {
}

std::unique_ptr<platf::avcodec_encode_device_t>
make_avcodec_encode_device(platf::pix_fmt_e pix_fmt) override {
make_avcodec_encode_device(platf::pix_fmt_e pix_fmt, bool yuv444in420) override {
if (yuv444in420) {
BOOST_LOG(error) << "Recombined YUV 4:4:4 is not supported";
return nullptr;
}

#ifdef SUNSHINE_BUILD_VAAPI
if (mem_type == platf::mem_type_e::vaapi) {
return va::make_avcodec_encode_device(width, height, false);
Expand Down Expand Up @@ -349,7 +354,12 @@ namespace wl {
}

std::unique_ptr<platf::avcodec_encode_device_t>
make_avcodec_encode_device(platf::pix_fmt_e pix_fmt) override {
make_avcodec_encode_device(platf::pix_fmt_e pix_fmt, bool yuv444in420) override {
if (yuv444in420) {
BOOST_LOG(error) << "Recombined YUV 4:4:4 is not supported";
return nullptr;
}

#ifdef SUNSHINE_BUILD_VAAPI
if (mem_type == platf::mem_type_e::vaapi) {
return va::make_avcodec_encode_device(width, height, 0, 0, true);
Expand Down
7 changes: 6 additions & 1 deletion src/platform/linux/x11grab.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -561,7 +561,12 @@ namespace platf {
}

std::unique_ptr<avcodec_encode_device_t>
make_avcodec_encode_device(pix_fmt_e pix_fmt) override {
make_avcodec_encode_device(pix_fmt_e pix_fmt, bool yuv444in420) override {
if (yuv444in420) {
BOOST_LOG(error) << "Recombined YUV 4:4:4 is not supported";
return nullptr;
}

#ifdef SUNSHINE_BUILD_VAAPI
if (mem_type == mem_type_e::vaapi) {
return va::make_avcodec_encode_device(width, height, false);
Expand Down
7 changes: 6 additions & 1 deletion src/platform/macos/display.mm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,12 @@
}

std::unique_ptr<avcodec_encode_device_t>
make_avcodec_encode_device(pix_fmt_e pix_fmt) override {
make_avcodec_encode_device(pix_fmt_e pix_fmt, bool yuv444in420) {
if (yuv444in420) {
BOOST_LOG(error) << "Recombined YUV 4:4:4 is not supported";
return nullptr;
}

if (pix_fmt == pix_fmt_e::yuv420p) {
av_capture.pixelFormat = kCVPixelFormatType_32BGRA;

Expand Down
11 changes: 8 additions & 3 deletions src/platform/windows/display.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,7 @@ namespace platf::dxgi {
get_supported_capture_formats() override;

std::unique_ptr<avcodec_encode_device_t>
make_avcodec_encode_device(pix_fmt_e pix_fmt) override;
make_avcodec_encode_device(pix_fmt_e pix_fmt, bool yuv444in420) override;

D3D11_MAPPED_SUBRESOURCE img_info;
texture2d_t texture;
Expand All @@ -295,11 +295,16 @@ namespace platf::dxgi {
bool
is_codec_supported(std::string_view name, const ::video::config_t &config) override;

bool
is_yuv444in420_supported() override {
return true;
}

std::unique_ptr<avcodec_encode_device_t>
make_avcodec_encode_device(pix_fmt_e pix_fmt) override;
make_avcodec_encode_device(pix_fmt_e pix_fmt, bool yuv444in420) override;

std::unique_ptr<nvenc_encode_device_t>
make_nvenc_encode_device(pix_fmt_e pix_fmt) override;
make_nvenc_encode_device(pix_fmt_e pix_fmt, bool yuv444in420) override;

std::atomic<uint32_t> next_image_id;
};
Expand Down
6 changes: 5 additions & 1 deletion src/platform/windows/display_ram.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -398,7 +398,11 @@ namespace platf::dxgi {
}

std::unique_ptr<avcodec_encode_device_t>
display_ram_t::make_avcodec_encode_device(pix_fmt_e pix_fmt) {
display_ram_t::make_avcodec_encode_device(pix_fmt_e pix_fmt, bool yuv444in420) {
if (yuv444in420) {
BOOST_LOG(error) << "Recombined YUV 4:4:4 is not supported";
return nullptr;
}
return std::make_unique<avcodec_encode_device_t>();
}

Expand Down
Loading