From a2f34ab3de3d8f18e13ae9960d76217b52ec0d8a Mon Sep 17 00:00:00 2001 From: ns6089 <61738816+ns6089@users.noreply.github.com> Date: Sun, 9 Jul 2023 17:05:27 +0300 Subject: [PATCH] Refactor and deduplicate colorspace logic --- CMakeLists.txt | 2 + src/nvenc/nvenc_utils.cpp | 62 +++++----- src/nvenc/nvenc_utils.h | 4 +- src/platform/common.h | 17 +-- src/platform/windows/display.h | 4 +- src/platform/windows/display_vram.cpp | 121 ++++++------------- src/video.cpp | 157 ++++++++---------------- src/video.h | 15 +-- src/video_colorspace.cpp | 166 ++++++++++++++++++++++++++ src/video_colorspace.h | 65 ++++++++++ 10 files changed, 358 insertions(+), 255 deletions(-) create mode 100644 src/video_colorspace.cpp create mode 100644 src/video_colorspace.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 17659f63c66..1b6826e204d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -525,6 +525,8 @@ set(SUNSHINE_TARGET_FILES src/stream.h src/video.cpp src/video.h + src/video_colorspace.cpp + src/video_colorspace.h src/input.cpp src/input.h src/audio.cpp diff --git a/src/nvenc/nvenc_utils.cpp b/src/nvenc/nvenc_utils.cpp index 7308f329ebf..4bc4693bb5e 100644 --- a/src/nvenc/nvenc_utils.cpp +++ b/src/nvenc/nvenc_utils.cpp @@ -33,45 +33,41 @@ namespace nvenc { } nvenc_colorspace_t - nvenc_colorspace_from_rtsp_csc(int csc, bool hdr) { - /* See video::config_t::encoderCscMode for details */ + nvenc_colorspace_from_sunshine_colorspace(const video::sunshine_colorspace_t &sunshine_colorspace) { + nvenc_colorspace_t colorspace; - nvenc_colorspace_t colorspace = {}; + switch (sunshine_colorspace.colorspace) { + case video::colorspace_e::rec601: + // Rec. 601 + colorspace.primaries = NV_ENC_VUI_COLOR_PRIMARIES_BT2020; + colorspace.tranfer_function = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_SMPTE2084; + colorspace.matrix = NV_ENC_VUI_MATRIX_COEFFS_BT2020_NCL; + break; - colorspace.full_range = (csc & 0x1); + case video::colorspace_e::rec709: + // Rec. 709 + colorspace.primaries = NV_ENC_VUI_COLOR_PRIMARIES_SMPTE170M; + colorspace.tranfer_function = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_SMPTE170M; + colorspace.matrix = NV_ENC_VUI_MATRIX_COEFFS_SMPTE170M; + break; - if (hdr) { - // Rec. 2020 + SMPTE 2084 PQ - colorspace.primaries = NV_ENC_VUI_COLOR_PRIMARIES_BT2020; - colorspace.tranfer_function = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_SMPTE2084; - colorspace.matrix = NV_ENC_VUI_MATRIX_COEFFS_BT2020_NCL; - } - else { - switch (csc >> 1) { - case 0: - default: - // Rec. 601 - colorspace.primaries = NV_ENC_VUI_COLOR_PRIMARIES_SMPTE170M; - colorspace.tranfer_function = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_SMPTE170M; - colorspace.matrix = NV_ENC_VUI_MATRIX_COEFFS_SMPTE170M; - break; - - case 1: - // Rec. 709 - colorspace.primaries = NV_ENC_VUI_COLOR_PRIMARIES_BT709; - colorspace.tranfer_function = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_BT709; - colorspace.matrix = NV_ENC_VUI_MATRIX_COEFFS_BT709; - break; + case video::colorspace_e::bt2020sdr: + // video::Rec. 2020 + colorspace.primaries = NV_ENC_VUI_COLOR_PRIMARIES_BT709; + colorspace.tranfer_function = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_BT709; + colorspace.matrix = NV_ENC_VUI_MATRIX_COEFFS_BT709; + break; - case 2: - // Rec. 2020 - colorspace.primaries = NV_ENC_VUI_COLOR_PRIMARIES_BT2020; - colorspace.tranfer_function = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_BT2020_10; - colorspace.matrix = NV_ENC_VUI_MATRIX_COEFFS_BT2020_NCL; - break; - } + case video::colorspace_e::bt2020: + // Rec. 2020 with ST 2084 perceptual quantizer + colorspace.primaries = NV_ENC_VUI_COLOR_PRIMARIES_BT2020; + colorspace.tranfer_function = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_BT2020_10; + colorspace.matrix = NV_ENC_VUI_MATRIX_COEFFS_BT2020_NCL; + break; } + colorspace.full_range = sunshine_colorspace.full_range; + return colorspace; } diff --git a/src/nvenc/nvenc_utils.h b/src/nvenc/nvenc_utils.h index f3a065e853c..73e23f3b35a 100644 --- a/src/nvenc/nvenc_utils.h +++ b/src/nvenc/nvenc_utils.h @@ -7,6 +7,7 @@ #include "nvenc_colorspace.h" #include "src/platform/common.h" +#include "src/video_colorspace.h" #include "third-party/nvenc/nvEncodeAPI.h" namespace nvenc { @@ -19,8 +20,7 @@ namespace nvenc { NV_ENC_BUFFER_FORMAT nvenc_format_from_sunshine_format(platf::pix_fmt_e format); - /* See video::config_t::encoderCscMode for details */ nvenc_colorspace_t - nvenc_colorspace_from_rtsp_csc(int csc, bool hdr); + nvenc_colorspace_from_sunshine_colorspace(const video::sunshine_colorspace_t &sunshine_colorspace); } // namespace nvenc diff --git a/src/platform/common.h b/src/platform/common.h index b7f83c8c7df..ec5d54c4842 100644 --- a/src/platform/common.h +++ b/src/platform/common.h @@ -14,6 +14,7 @@ #include "src/nvenc/nvenc_encoded_frame.h" #include "src/thread_safe.h" #include "src/utility.h" +#include "src/video_colorspace.h" extern "C" { #include @@ -328,9 +329,6 @@ namespace platf { virtual int convert(platf::img_t &img) = 0; - virtual void - set_colorspace(std::uint32_t colorspace, std::uint32_t color_range) = 0; - virtual void request_idr_frame() = 0; @@ -341,6 +339,8 @@ namespace platf { invalidate_ref_frames(int64_t first_frame, int64_t last_frame) { BOOST_LOG(error) << "Encoding stream doesn't support reference frame invalidation"; } + + video::sunshine_colorspace_t colorspace; }; struct avcodec_encoding_stream_t: encoding_stream_t { @@ -352,11 +352,6 @@ namespace platf { return -1; } - void - set_colorspace(std::uint32_t colorspace, std::uint32_t color_range) override { - // TODO: leave as pure virtual? - } - void request_idr_frame() override { if (frame) { @@ -464,12 +459,12 @@ namespace platf { dummy_img(img_t *img) = 0; virtual std::unique_ptr - make_avcodec_encoding_stream(pix_fmt_e pix_fmt) { - return std::make_unique(); + make_avcodec_encoding_stream(pix_fmt_e pix_fmt, const video::sunshine_colorspace_t &colorspace) { + return nullptr; } virtual std::unique_ptr - make_nvenc_encoding_stream(const video::config_t &config, pix_fmt_e pix_fmt) { + make_nvenc_encoding_stream(const video::config_t &config, pix_fmt_e pix_fmt, const video::sunshine_colorspace_t &colorspace) { return nullptr; } diff --git a/src/platform/windows/display.h b/src/platform/windows/display.h index 610225eccaf..95e14cb7d8d 100644 --- a/src/platform/windows/display.h +++ b/src/platform/windows/display.h @@ -217,10 +217,10 @@ namespace platf::dxgi { init(const ::video::config_t &config, const std::string &display_name); std::unique_ptr - make_avcodec_encoding_stream(pix_fmt_e pix_fmt) override; + make_avcodec_encoding_stream(pix_fmt_e pix_fmt, const ::video::sunshine_colorspace_t &colorspace) override; std::unique_ptr - make_nvenc_encoding_stream(const ::video::config_t &config, pix_fmt_e pix_fmt) override; + make_nvenc_encoding_stream(const ::video::config_t &config, pix_fmt_e pix_fmt, const ::video::sunshine_colorspace_t &colorspace) override; sampler_state_t sampler_linear; diff --git a/src/platform/windows/display_vram.cpp b/src/platform/windows/display_vram.cpp index 2624ce4a67b..6b8e8f955a7 100644 --- a/src/platform/windows/display_vram.cpp +++ b/src/platform/windows/display_vram.cpp @@ -416,39 +416,6 @@ namespace platf::dxgi { return 0; } - void - set_colorspace(std::uint32_t colorspace, std::uint32_t color_range) { - switch (colorspace) { - case 5: // SWS_CS_SMPTE170M - color_p = &::video::colors[0]; - break; - case 1: // SWS_CS_ITU709 - color_p = &::video::colors[2]; - break; - case 9: // SWS_CS_BT2020 - color_p = &::video::colors[4]; - break; - default: - BOOST_LOG(warning) << "Colorspace: ["sv << colorspace << "] not yet supported: switching to default"sv; - color_p = &::video::colors[0]; - }; - - if (color_range > 1) { - // Full range - ++color_p; - } - - auto color_matrix = make_buffer(device.get(), *color_p); - if (!color_matrix) { - BOOST_LOG(warning) << "Failed to create color matrix"sv; - return; - } - - device_ctx->VSSetConstantBuffers(0, 1, &info_scene); - device_ctx->PSSetConstantBuffers(0, 1, &color_matrix); - this->color_matrix = std::move(color_matrix); - } - int init_output(ID3D11Texture2D *frame_texture, int width, int height) { // The underlying frame pool owns the texture, so we must reference it for ourselves @@ -510,7 +477,7 @@ namespace platf::dxgi { } int - init(std::shared_ptr display, adapter_t::pointer adapter_p, pix_fmt_e pix_fmt) { + init(std::shared_ptr display, adapter_t::pointer adapter_p, pix_fmt_e pix_fmt, const ::video::sunshine_colorspace_t &colorspace) { D3D_FEATURE_LEVEL featureLevels[] { D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0, @@ -605,7 +572,13 @@ namespace platf::dxgi { return -1; } - color_matrix = make_buffer(device.get(), ::video::colors[0]); + auto default_color_vectors = ::video::color_vectors_from_colorspace(::video::colorspace_e::rec601, false); + if (!default_color_vectors) { + BOOST_LOG(error) << "Missing color vectors for Rec. 601"sv; + return -1; + } + + color_matrix = make_buffer(device.get(), *default_color_vectors); if (!color_matrix) { BOOST_LOG(error) << "Failed to create color matrix buffer"sv; return -1; @@ -650,6 +623,23 @@ namespace platf::dxgi { device_ctx->PSSetSamplers(0, 1, &sampler_linear); device_ctx->IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLESTRIP); + auto color_vectors = ::video::color_vectors_from_colorspace(colorspace); + + if (!color_vectors) { + BOOST_LOG(error) << "No vector data for colorspace"sv; + return -1; + } + + auto color_matrix = make_buffer(device.get(), *color_vectors); + if (!color_matrix) { + BOOST_LOG(warning) << "Failed to create color matrix"sv; + return -1; + } + + device_ctx->VSSetConstantBuffers(0, 1, &info_scene); + device_ctx->PSSetConstantBuffers(0, 1, &color_matrix); + this->color_matrix = std::move(color_matrix); + return 0; } @@ -762,8 +752,8 @@ namespace platf::dxgi { class d3d_avcodec_encoding_stream_t: public avcodec_encoding_stream_t { public: int - init(std::shared_ptr display, adapter_t::pointer adapter_p, pix_fmt_e pix_fmt) { - int result = base.init(display, adapter_p, pix_fmt); + init(std::shared_ptr display, adapter_t::pointer adapter_p, pix_fmt_e pix_fmt, const ::video::sunshine_colorspace_t &colorspace) { + int result = base.init(display, adapter_p, pix_fmt, colorspace); data = base.device.get(); return result; } @@ -773,11 +763,6 @@ namespace platf::dxgi { return base.convert(img_base); } - void - set_colorspace(std::uint32_t colorspace, std::uint32_t color_range) override { - base.set_colorspace(colorspace, color_range); - } - void init_hwframes(AVHWFramesContext *frames) override { // We may be called with a QSV or D3D11VA context @@ -859,48 +844,18 @@ namespace platf::dxgi { class d3d_nvenc_encoding_stream_t: public nvenc_encoding_stream_t { public: bool - init(std::shared_ptr display, adapter_t::pointer adapter_p, const ::video::config_t &client_config, pix_fmt_e pix_fmt) { + init(std::shared_ptr display, adapter_t::pointer adapter_p, const ::video::config_t &client_config, pix_fmt_e pix_fmt, const ::video::sunshine_colorspace_t &colorspace) { auto buffer_format = nvenc::nvenc_format_from_sunshine_format(pix_fmt); if (buffer_format == NV_ENC_BUFFER_FORMAT_UNDEFINED) { BOOST_LOG(error) << "Unexpected pixel format for NvENC ["sv << from_pix_fmt(pix_fmt) << ']'; return false; } - if (base.init(display, adapter_p, pix_fmt)) return false; - - // TODO: add sunshine colorspace/primaries/tranfer/matrix constants and translate those to avcodec, nvenc, whatever - { - int color_range = (client_config.encoderCscMode & 0x1) ? 2 : 1; - int color_space; - if (client_config.dynamicRange && display->is_hdr()) { - // When HDR is active, that overrides the colorspace the client requested - color_space = 9; - } - else { - switch (client_config.encoderCscMode >> 1) { - case 0: - default: - // BT.601 - color_space = 5; - break; - - case 1: - // BT.709 - color_space = 1; - break; - - case 2: - // BT.2020 - color_space = 9; - break; - } - } - set_colorspace(color_space, color_range); - } + if (base.init(display, adapter_p, pix_fmt, colorspace)) return false; nvenc = std::make_unique(base.device.get(), client_config.width, client_config.height, buffer_format); - auto nvenc_colorspace = nvenc::nvenc_colorspace_from_rtsp_csc(client_config.encoderCscMode, client_config.dynamicRange && display->is_hdr()); + auto nvenc_colorspace = nvenc::nvenc_colorspace_from_sunshine_colorspace(colorspace); nvenc::nvenc_config nvenc_config; @@ -921,11 +876,6 @@ namespace platf::dxgi { return base.convert(img_base); } - void - set_colorspace(std::uint32_t colorspace, std::uint32_t color_range) override { - base.set_colorspace(colorspace, color_range); - } - void request_idr_frame() override { force_idr = true; @@ -1585,7 +1535,7 @@ namespace platf::dxgi { } std::unique_ptr - display_vram_t::make_avcodec_encoding_stream(pix_fmt_e pix_fmt) { + display_vram_t::make_avcodec_encoding_stream(pix_fmt_e pix_fmt, const ::video::sunshine_colorspace_t &colorspace) { if (pix_fmt != platf::pix_fmt_e::nv12 && pix_fmt != platf::pix_fmt_e::p010) { BOOST_LOG(error) << "display_vram_t doesn't support pixel format ["sv << from_pix_fmt(pix_fmt) << ']'; @@ -1594,10 +1544,7 @@ namespace platf::dxgi { auto hwdevice = std::make_unique(); - auto ret = hwdevice->init( - shared_from_this(), - adapter.get(), - pix_fmt); + auto ret = hwdevice->init(shared_from_this(), adapter.get(), pix_fmt, colorspace); if (ret) { return nullptr; @@ -1607,9 +1554,9 @@ namespace platf::dxgi { } std::unique_ptr - display_vram_t::make_nvenc_encoding_stream(const ::video::config_t &config, pix_fmt_e pix_fmt) { + display_vram_t::make_nvenc_encoding_stream(const ::video::config_t &config, pix_fmt_e pix_fmt, const ::video::sunshine_colorspace_t &colorspace) { auto encoding_stream = std::make_unique(); - if (!encoding_stream->init(shared_from_this(), adapter.get(), config, pix_fmt)) { + if (!encoding_stream->init(shared_from_this(), adapter.get(), config, pix_fmt, colorspace)) { return nullptr; } return encoding_stream; diff --git a/src/video.cpp b/src/video.cpp index d7f63ee7a44..423a6b71536 100644 --- a/src/video.cpp +++ b/src/video.cpp @@ -155,14 +155,6 @@ namespace video { return 0; } - void - set_colorspace(std::uint32_t colorspace, std::uint32_t color_range) override { - sws_setColorspaceDetails(sws.get(), - sws_getCoefficients(SWS_CS_DEFAULT), 0, - sws_getCoefficients(colorspace), color_range - 1, - 0, 1 << 16, 1 << 16); - } - /** * When preserving aspect ratio, ensure that padding is black */ @@ -206,7 +198,7 @@ namespace video { } int - init(int in_width, int in_height, AVFrame *frame, AVPixelFormat format, bool hardware) { + init(int in_width, int in_height, AVFrame *frame, AVPixelFormat format, bool hardware, const avcodec_colorspace_t &colorspace) { // If the device used is hardware, yet the image resides on main memory if (hardware) { sw_frame.reset(av_frame_alloc()); @@ -243,6 +235,12 @@ namespace video { SWS_LANCZOS | SWS_ACCURATE_RND, nullptr, nullptr, nullptr)); + if (sws) { + int range_correction = (colorspace.range == AVCOL_RANGE_JPEG); + sws_setColorspaceDetails(sws.get(), sws_getCoefficients(SWS_CS_DEFAULT), 0, + sws_getCoefficients(colorspace.software_format), range_correction, 0, 1 << 16, 1 << 16); + } + return sws ? 0 : -1; } @@ -1234,58 +1232,18 @@ namespace video { ctx->flags |= (AV_CODEC_FLAG_CLOSED_GOP | AV_CODEC_FLAG_LOW_DELAY); ctx->flags2 |= AV_CODEC_FLAG2_FAST; - ctx->color_range = (config.encoderCscMode & 0x1) ? AVCOL_RANGE_JPEG : AVCOL_RANGE_MPEG; - - int sws_color_space; - if (config.dynamicRange && disp->is_hdr()) { - // When HDR is active, that overrides the colorspace the client requested - BOOST_LOG(info) << "HDR color coding [Rec. 2020 + SMPTE 2084 PQ]"sv; - ctx->color_primaries = AVCOL_PRI_BT2020; - ctx->color_trc = AVCOL_TRC_SMPTE2084; - ctx->colorspace = AVCOL_SPC_BT2020_NCL; - sws_color_space = SWS_CS_BT2020; - } - else { - switch (config.encoderCscMode >> 1) { - case 0: - default: - // Rec. 601 - BOOST_LOG(info) << "SDR color coding [Rec. 601]"sv; - ctx->color_primaries = AVCOL_PRI_SMPTE170M; - ctx->color_trc = AVCOL_TRC_SMPTE170M; - ctx->colorspace = AVCOL_SPC_SMPTE170M; - sws_color_space = SWS_CS_SMPTE170M; - break; + auto colorspace = colorspace_from_rtsp_csc(config.encoderCscMode, config.dynamicRange, disp->is_hdr()); - case 1: - // Rec. 709 - BOOST_LOG(info) << "SDR color coding [Rec. 709]"sv; - ctx->color_primaries = AVCOL_PRI_BT709; - ctx->color_trc = AVCOL_TRC_BT709; - ctx->colorspace = AVCOL_SPC_BT709; - sws_color_space = SWS_CS_ITU709; - break; + auto avcodec_colorspace = avcodec_colorspace_from_sunshine_colorspace(colorspace); - case 2: - // Rec. 2020 - BOOST_LOG(info) << "SDR color coding [Rec. 2020]"sv; - ctx->color_primaries = AVCOL_PRI_BT2020; - ctx->color_trc = AVCOL_TRC_BT2020_10; - ctx->colorspace = AVCOL_SPC_BT2020_NCL; - sws_color_space = SWS_CS_BT2020; - break; - } - } + ctx->color_range = avcodec_colorspace.range; + BOOST_LOG(info) << "Color range: ["sv << (ctx->color_range == AVCOL_RANGE_JPEG ? "JPEG"sv : "MPEG"sv) << ']'; - BOOST_LOG(info) << "Color range: ["sv << ((config.encoderCscMode & 0x1) ? "JPEG"sv : "MPEG"sv) << ']'; + ctx->color_primaries = avcodec_colorspace.primaries; + ctx->color_trc = avcodec_colorspace.transfer_function; + ctx->colorspace = avcodec_colorspace.matrix; - AVPixelFormat sw_fmt; - if (config.dynamicRange == 0) { - sw_fmt = encoder_details->avcodec_static_pix_fmt; - } - else { - sw_fmt = encoder_details->avcodec_dynamic_pix_fmt; - } + auto sw_fmt = colorspace_is_dynamic(colorspace) ? encoder_details->avcodec_static_pix_fmt : encoder_details->avcodec_dynamic_pix_fmt; // Used by cbs::make_sps_hevc ctx->sw_pix_fmt = sw_fmt; @@ -1435,7 +1393,7 @@ namespace video { frame->height = ctx->height; // Attach HDR metadata to the AVFrame - if (config.dynamicRange && disp->is_hdr()) { + if (colorspace_needs_dynamic_metadata(colorspace)) { SS_HDR_METADATA hdr_metadata; if (disp->get_hdr_metadata(hdr_metadata)) { auto mdm = av_mastering_display_metadata_create_side_data(frame.get()); @@ -1463,6 +1421,9 @@ namespace video { clm->MaxFALL = hdr_metadata.maxFrameAverageLightLevel; } } + else { + BOOST_LOG(error) << "Couldn't get display hdr metadata when colorspace selection indicates it should have one"; + } } std::unique_ptr encoding_stream_final; @@ -1470,7 +1431,7 @@ namespace video { if (!encoding_stream->data) { auto software_encoding_stream = std::make_unique(); - if (software_encoding_stream->init(width, height, frame.get(), sw_fmt, hardware)) { + if (software_encoding_stream->init(width, height, frame.get(), sw_fmt, hardware, avcodec_colorspace)) { return std::nullopt; } @@ -1484,8 +1445,6 @@ namespace video { return std::nullopt; } - encoding_stream_final->set_colorspace(sws_color_space, ctx->color_range); - session_t session { std::move(ctx), std::move(encoding_stream_final), @@ -1626,15 +1585,24 @@ namespace video { } std::unique_ptr - make_encoding_stream(platf::display_t &disp, const encoder_t &encoder, const config_t &config, platf::pix_fmt_e pix_fmt) { + make_encoding_stream(platf::display_t &disp, const encoder_t &encoder, const config_t &config) { + std::unique_ptr result; + + auto colorspace = colorspace_from_rtsp_csc(config.encoderCscMode, config.dynamicRange, disp.is_hdr()); + auto pix_fmt = colorspace_is_dynamic(colorspace) ? encoder.details->dynamic_pix_fmt : encoder.details->static_pix_fmt; + if (dynamic_cast(encoder.details.get())) { - return disp.make_avcodec_encoding_stream(pix_fmt); + result = disp.make_avcodec_encoding_stream(pix_fmt, colorspace); } else if (dynamic_cast(encoder.details.get())) { - return disp.make_nvenc_encoding_stream(config, pix_fmt); + result = disp.make_nvenc_encoding_stream(config, pix_fmt, colorspace); + } + + if (result) { + result->colorspace = colorspace; } - return nullptr; + return result; } std::optional @@ -1643,8 +1611,7 @@ namespace video { encode_session.ctx = &ctx; - auto pix_fmt = ctx.config.dynamicRange == 0 ? encoder.details->static_pix_fmt : encoder.details->dynamic_pix_fmt; - auto encoding_stream = make_encoding_stream(*disp, encoder, ctx.config, pix_fmt); + auto encoding_stream = make_encoding_stream(*disp, encoder, ctx.config); if (!encoding_stream) { return std::nullopt; } @@ -1654,9 +1621,13 @@ namespace video { // Update client with our current HDR display state hdr_info_t hdr_info = std::make_unique(false); - if (ctx.config.dynamicRange && disp->is_hdr()) { - disp->get_hdr_metadata(hdr_info->metadata); - hdr_info->enabled = true; + if (colorspace_needs_dynamic_metadata(encoding_stream->colorspace)) { + if (disp->get_hdr_metadata(hdr_info->metadata)) { + hdr_info->enabled = true; + } + else { + BOOST_LOG(error) << "Couldn't get display hdr metadata when colorspace selection indicates it should have one"; + } } ctx.hdr_events->raise(std::move(hdr_info)); @@ -1904,8 +1875,8 @@ namespace video { } auto &encoder = *chosen_encoder; - auto pix_fmt = config.dynamicRange == 0 ? encoder.details->static_pix_fmt : encoder.details->dynamic_pix_fmt; - auto encoding_stream = make_encoding_stream(*display, encoder, config, pix_fmt); + + auto encoding_stream = make_encoding_stream(*display, encoder, config); if (!encoding_stream) { return; } @@ -1915,9 +1886,13 @@ namespace video { // Update client with our current HDR display state hdr_info_t hdr_info = std::make_unique(false); - if (config.dynamicRange && display->is_hdr()) { - display->get_hdr_metadata(hdr_info->metadata); - hdr_info->enabled = true; + if (colorspace_needs_dynamic_metadata(encoding_stream->colorspace)) { + if (display->get_hdr_metadata(hdr_info->metadata)) { + hdr_info->enabled = true; + } + else { + BOOST_LOG(error) << "Couldn't get display hdr metadata when colorspace selection indicates it should have one"; + } } hdr_event->raise(std::move(hdr_info)); @@ -1973,8 +1948,7 @@ namespace video { return -1; } - auto pix_fmt = config.dynamicRange == 0 ? encoder.details->static_pix_fmt : encoder.details->dynamic_pix_fmt; - auto initial_encoding_stream = make_encoding_stream(*disp, encoder, config, pix_fmt); + auto initial_encoding_stream = make_encoding_stream(*disp, encoder, config); if (!initial_encoding_stream) { return -1; } @@ -2417,33 +2391,4 @@ namespace video { return platf::pix_fmt_e::unknown; } - color_t - make_color_matrix(float Cr, float Cb, const float2 &range_Y, const float2 &range_UV) { - float Cg = 1.0f - Cr - Cb; - - float Cr_i = 1.0f - Cr; - float Cb_i = 1.0f - Cb; - - float shift_y = range_Y[0] / 255.0f; - float shift_uv = range_UV[0] / 255.0f; - - float scale_y = (range_Y[1] - range_Y[0]) / 255.0f; - float scale_uv = (range_UV[1] - range_UV[0]) / 255.0f; - return { - { Cr, Cg, Cb, 0.0f }, - { -(Cr * 0.5f / Cb_i), -(Cg * 0.5f / Cb_i), 0.5f, 0.5f }, - { 0.5f, -(Cg * 0.5f / Cr_i), -(Cb * 0.5f / Cr_i), 0.5f }, - { scale_y, shift_y }, - { scale_uv, shift_uv }, - }; - } - - color_t colors[] { - make_color_matrix(0.299f, 0.114f, { 16.0f, 235.0f }, { 16.0f, 240.0f }), // BT601 MPEG - make_color_matrix(0.299f, 0.114f, { 0.0f, 255.0f }, { 0.0f, 255.0f }), // BT601 JPEG - make_color_matrix(0.2126f, 0.0722f, { 16.0f, 235.0f }, { 16.0f, 240.0f }), // BT709 MPEG - make_color_matrix(0.2126f, 0.0722f, { 0.0f, 255.0f }, { 0.0f, 255.0f }), // BT709 JPEG - make_color_matrix(0.2627f, 0.0593f, { 16.0f, 235.0f }, { 16.0f, 240.0f }), // BT2020 MPEG - make_color_matrix(0.2627f, 0.0593f, { 0.0f, 255.0f }, { 0.0f, 255.0f }), // BT2020 JPEG - }; } // namespace video diff --git a/src/video.h b/src/video.h index 80c9cd364dd..c0499b2e16b 100644 --- a/src/video.h +++ b/src/video.h @@ -7,6 +7,7 @@ #include "input.h" #include "platform/common.h" #include "thread_safe.h" +#include "video_colorspace.h" extern "C" { #include @@ -140,20 +141,6 @@ namespace video { int dynamicRange; // 0 - SDR, 1 - HDR }; - using float4 = float[4]; - using float3 = float[3]; - using float2 = float[2]; - - struct alignas(16) color_t { - float4 color_vec_y; - float4 color_vec_u; - float4 color_vec_v; - float2 range_y; - float2 range_uv; - }; - - extern color_t colors[6]; - extern int active_hevc_mode; extern bool last_encoder_probe_supported_invalidate_ref_frames; diff --git a/src/video_colorspace.cpp b/src/video_colorspace.cpp new file mode 100644 index 00000000000..7d870dcbd66 --- /dev/null +++ b/src/video_colorspace.cpp @@ -0,0 +1,166 @@ +#include "video_colorspace.h" + +extern "C" { +#include +} + +namespace video { + + bool + colorspace_is_dynamic(colorspace_e colorspace) { + return colorspace == colorspace_e::bt2020sdr || colorspace == colorspace_e::bt2020; + } + + bool + colorspace_is_dynamic(const sunshine_colorspace_t &colorspace) { + return colorspace_is_dynamic(colorspace.colorspace); + } + + bool + colorspace_needs_dynamic_metadata(colorspace_e colorspace) { + return colorspace == colorspace_e::bt2020; + } + + bool + colorspace_needs_dynamic_metadata(const sunshine_colorspace_t &colorspace) { + return colorspace_needs_dynamic_metadata(colorspace.colorspace); + } + + sunshine_colorspace_t + colorspace_from_rtsp_csc(int csc, bool hdr_requested, bool hdr_display) { + /* See video::config_t::encoderCscMode for details */ + + sunshine_colorspace_t colorspace; + + // When HDR is active, that overrides the colorspace the client requested + if (hdr_requested && hdr_display) { + // Rec. 2020 with ST 2084 perceptual quantizer + colorspace.colorspace = colorspace_e::bt2020; + } + else { + switch (csc >> 1) { + case 0: + default: + // Rec. 601 + colorspace.colorspace = colorspace_e::rec601; + break; + + case 1: + // Rec. 709 + colorspace.colorspace = colorspace_e::rec709; + break; + + case 2: + // Rec. 2020 + colorspace.colorspace = colorspace_e::bt2020sdr; + break; + } + } + + colorspace.full_range = (csc & 0x1); + + return colorspace; + } + + avcodec_colorspace_t + avcodec_colorspace_from_sunshine_colorspace(const sunshine_colorspace_t &sunshine_colorspace) { + avcodec_colorspace_t avcodec_colorspace; + + switch (sunshine_colorspace.colorspace) { + case colorspace_e::rec601: + // Rec. 601 + avcodec_colorspace.primaries = AVCOL_PRI_SMPTE170M; + avcodec_colorspace.transfer_function = AVCOL_TRC_SMPTE170M; + avcodec_colorspace.matrix = AVCOL_SPC_SMPTE170M; + avcodec_colorspace.software_format = SWS_CS_SMPTE170M; + break; + + case colorspace_e::rec709: + // Rec. 709 + avcodec_colorspace.primaries = AVCOL_PRI_BT709; + avcodec_colorspace.transfer_function = AVCOL_TRC_BT709; + avcodec_colorspace.matrix = AVCOL_SPC_BT709; + avcodec_colorspace.software_format = SWS_CS_ITU709; + break; + + case colorspace_e::bt2020sdr: + // Rec. 2020 + avcodec_colorspace.primaries = AVCOL_PRI_BT2020; + avcodec_colorspace.transfer_function = AVCOL_TRC_BT2020_10; + avcodec_colorspace.matrix = AVCOL_SPC_BT2020_NCL; + avcodec_colorspace.software_format = SWS_CS_BT2020; + break; + + case colorspace_e::bt2020: + // Rec. 2020 with ST 2084 perceptual quantizer + avcodec_colorspace.primaries = AVCOL_PRI_BT2020; + avcodec_colorspace.transfer_function = AVCOL_TRC_SMPTE2084; + avcodec_colorspace.matrix = AVCOL_SPC_BT2020_NCL; + avcodec_colorspace.software_format = SWS_CS_BT2020; + break; + } + + avcodec_colorspace.range = sunshine_colorspace.full_range ? AVCOL_RANGE_JPEG : AVCOL_RANGE_MPEG; + + return avcodec_colorspace; + } + + const color_t * + color_vectors_from_colorspace(const sunshine_colorspace_t &colorspace) { + return color_vectors_from_colorspace(colorspace.colorspace, colorspace.full_range); + } + + const color_t * + color_vectors_from_colorspace(colorspace_e colorspace, bool full_range) { + auto make_color_matrix = [](float Cr, float Cb, const float2 &range_Y, const float2 &range_UV) -> color_t { + float Cg = 1.0f - Cr - Cb; + + float Cr_i = 1.0f - Cr; + float Cb_i = 1.0f - Cb; + + float shift_y = range_Y[0] / 255.0f; + float shift_uv = range_UV[0] / 255.0f; + + float scale_y = (range_Y[1] - range_Y[0]) / 255.0f; + float scale_uv = (range_UV[1] - range_UV[0]) / 255.0f; + return { + { Cr, Cg, Cb, 0.0f }, + { -(Cr * 0.5f / Cb_i), -(Cg * 0.5f / Cb_i), 0.5f, 0.5f }, + { 0.5f, -(Cg * 0.5f / Cr_i), -(Cb * 0.5f / Cr_i), 0.5f }, + { scale_y, shift_y }, + { scale_uv, shift_uv }, + }; + }; + + static const color_t colors[] { + make_color_matrix(0.299f, 0.114f, { 16.0f, 235.0f }, { 16.0f, 240.0f }), // BT601 MPEG + make_color_matrix(0.299f, 0.114f, { 0.0f, 255.0f }, { 0.0f, 255.0f }), // BT601 JPEG + make_color_matrix(0.2126f, 0.0722f, { 16.0f, 235.0f }, { 16.0f, 240.0f }), // BT709 MPEG + make_color_matrix(0.2126f, 0.0722f, { 0.0f, 255.0f }, { 0.0f, 255.0f }), // BT709 JPEG + make_color_matrix(0.2627f, 0.0593f, { 16.0f, 235.0f }, { 16.0f, 240.0f }), // BT2020 MPEG + make_color_matrix(0.2627f, 0.0593f, { 0.0f, 255.0f }, { 0.0f, 255.0f }), // BT2020 JPEG + }; + + const color_t *result = nullptr; + + switch (colorspace) { + case colorspace_e::rec601: + result = &colors[0]; + break; + case colorspace_e::rec709: + result = &colors[2]; + break; + case colorspace_e::bt2020: + case colorspace_e::bt2020sdr: + result = &colors[4]; + break; + }; + + if (result && full_range) { + result++; + } + + return result; + } + +} // namespace video diff --git a/src/video_colorspace.h b/src/video_colorspace.h new file mode 100644 index 00000000000..9b547018f64 --- /dev/null +++ b/src/video_colorspace.h @@ -0,0 +1,65 @@ +#pragma once + +extern "C" { +#include +} + +namespace video { + + enum class colorspace_e { + rec601, + rec709, + bt2020sdr, + bt2020, + }; + + struct sunshine_colorspace_t { + colorspace_e colorspace; + bool full_range; + }; + + bool + colorspace_is_dynamic(colorspace_e colorspace); + + bool + colorspace_is_dynamic(const sunshine_colorspace_t &colorspace); + + bool + colorspace_needs_dynamic_metadata(colorspace_e colorspace); + + bool + colorspace_needs_dynamic_metadata(const sunshine_colorspace_t &colorspace); + + sunshine_colorspace_t + colorspace_from_rtsp_csc(int csc, bool hdr_requested, bool hdr_display); + + struct avcodec_colorspace_t { + AVColorPrimaries primaries; + AVColorTransferCharacteristic transfer_function; + AVColorSpace matrix; + AVColorRange range; + int software_format; + }; + + avcodec_colorspace_t + avcodec_colorspace_from_sunshine_colorspace(const sunshine_colorspace_t &sunshine_colorspace); + + using float4 = float[4]; + using float3 = float[3]; + using float2 = float[2]; + + struct alignas(16) color_t { + float4 color_vec_y; + float4 color_vec_u; + float4 color_vec_v; + float2 range_y; + float2 range_uv; + }; + + const color_t * + color_vectors_from_colorspace(const sunshine_colorspace_t &colorspace); + + const color_t * + color_vectors_from_colorspace(colorspace_e colorspace, bool full_range); + +} // namespace video