Skip to content

Commit

Permalink
Refactor and deduplicate colorspace logic
Browse files Browse the repository at this point in the history
  • Loading branch information
@ns6089
ns6089 committed Jul 9, 2023
1 parent f93642a commit 6836c6e
Show file tree
Hide file tree
Showing 10 changed files with 358 additions and 255 deletions.
2 changes: 2 additions & 0 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -525,6 +525,8 @@ set(SUNSHINE_TARGET_FILES
src/stream.h
src/video.cpp
src/video.h
src/video_colorspace.cpp
src/video_colorspace.h
src/input.cpp
src/input.h
src/audio.cpp
Expand Down
62 changes: 29 additions & 33 deletions src/nvenc/nvenc_utils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,45 +33,41 @@ namespace nvenc {
}

nvenc_colorspace_t
nvenc_colorspace_from_rtsp_csc(int csc, bool hdr) {
/* See video::config_t::encoderCscMode for details */
nvenc_colorspace_from_sunshine_colorspace(const video::sunshine_colorspace_t &sunshine_colorspace) {
nvenc_colorspace_t colorspace;

nvenc_colorspace_t colorspace = {};
switch (sunshine_colorspace.colorspace) {
case video::colorspace_e::rec601:
// Rec. 601
colorspace.primaries = NV_ENC_VUI_COLOR_PRIMARIES_BT2020;
colorspace.tranfer_function = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_SMPTE2084;
colorspace.matrix = NV_ENC_VUI_MATRIX_COEFFS_BT2020_NCL;
break;

colorspace.full_range = (csc & 0x1);
case video::colorspace_e::rec709:
// Rec. 709
colorspace.primaries = NV_ENC_VUI_COLOR_PRIMARIES_SMPTE170M;
colorspace.tranfer_function = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_SMPTE170M;
colorspace.matrix = NV_ENC_VUI_MATRIX_COEFFS_SMPTE170M;
break;

if (hdr) {
// Rec. 2020 + SMPTE 2084 PQ
colorspace.primaries = NV_ENC_VUI_COLOR_PRIMARIES_BT2020;
colorspace.tranfer_function = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_SMPTE2084;
colorspace.matrix = NV_ENC_VUI_MATRIX_COEFFS_BT2020_NCL;
}
else {
switch (csc >> 1) {
case 0:
default:
// Rec. 601
colorspace.primaries = NV_ENC_VUI_COLOR_PRIMARIES_SMPTE170M;
colorspace.tranfer_function = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_SMPTE170M;
colorspace.matrix = NV_ENC_VUI_MATRIX_COEFFS_SMPTE170M;
break;

case 1:
// Rec. 709
colorspace.primaries = NV_ENC_VUI_COLOR_PRIMARIES_BT709;
colorspace.tranfer_function = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_BT709;
colorspace.matrix = NV_ENC_VUI_MATRIX_COEFFS_BT709;
break;
case video::colorspace_e::bt2020sdr:
// video::Rec. 2020
colorspace.primaries = NV_ENC_VUI_COLOR_PRIMARIES_BT709;
colorspace.tranfer_function = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_BT709;
colorspace.matrix = NV_ENC_VUI_MATRIX_COEFFS_BT709;
break;

case 2:
// Rec. 2020
colorspace.primaries = NV_ENC_VUI_COLOR_PRIMARIES_BT2020;
colorspace.tranfer_function = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_BT2020_10;
colorspace.matrix = NV_ENC_VUI_MATRIX_COEFFS_BT2020_NCL;
break;
}
case video::colorspace_e::bt2020:
// Rec. 2020 with ST 2084 perceptual quantizer
colorspace.primaries = NV_ENC_VUI_COLOR_PRIMARIES_BT2020;
colorspace.tranfer_function = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_BT2020_10;
colorspace.matrix = NV_ENC_VUI_MATRIX_COEFFS_BT2020_NCL;
break;
}

colorspace.full_range = sunshine_colorspace.full_range;

return colorspace;
}

Expand Down
4 changes: 2 additions & 2 deletions src/nvenc/nvenc_utils.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#include "nvenc_colorspace.h"

#include "src/platform/common.h"
#include "src/video_colorspace.h"
#include "third-party/nvenc/nvEncodeAPI.h"

namespace nvenc {
Expand All @@ -19,8 +20,7 @@ namespace nvenc {
NV_ENC_BUFFER_FORMAT
nvenc_format_from_sunshine_format(platf::pix_fmt_e format);

/* See video::config_t::encoderCscMode for details */
nvenc_colorspace_t
nvenc_colorspace_from_rtsp_csc(int csc, bool hdr);
nvenc_colorspace_from_sunshine_colorspace(const video::sunshine_colorspace_t &sunshine_colorspace);

} // namespace nvenc
17 changes: 6 additions & 11 deletions src/platform/common.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#include "src/nvenc/nvenc_encoded_frame.h"
#include "src/thread_safe.h"
#include "src/utility.h"
#include "src/video_colorspace.h"

extern "C" {
#include <libavutil/frame.h>
Expand Down Expand Up @@ -226,9 +227,6 @@ namespace platf {
virtual int
convert(platf::img_t &img) = 0;

virtual void
set_colorspace(std::uint32_t colorspace, std::uint32_t color_range) = 0;

virtual void
request_idr_frame() = 0;

Expand All @@ -239,6 +237,8 @@ namespace platf {
invalidate_ref_frames(int64_t first_frame, int64_t last_frame) {
BOOST_LOG(error) << "Encoding stream doesn't support reference frame invalidation";
}

video::sunshine_colorspace_t colorspace;
};

struct avcodec_encoding_stream_t: encoding_stream_t {
Expand All @@ -250,11 +250,6 @@ namespace platf {
return -1;
}

void
set_colorspace(std::uint32_t colorspace, std::uint32_t color_range) override {
// TODO: leave as pure virtual?
}

void
request_idr_frame() override {
if (frame) {
Expand Down Expand Up @@ -362,12 +357,12 @@ namespace platf {
dummy_img(img_t *img) = 0;

virtual std::unique_ptr<avcodec_encoding_stream_t>
make_avcodec_encoding_stream(pix_fmt_e pix_fmt) {
return std::make_unique<avcodec_encoding_stream_t>();
make_avcodec_encoding_stream(pix_fmt_e pix_fmt, const video::sunshine_colorspace_t &colorspace) {
return nullptr;
}

virtual std::unique_ptr<nvenc_encoding_stream_t>
make_nvenc_encoding_stream(const video::config_t &config, pix_fmt_e pix_fmt) {
make_nvenc_encoding_stream(const video::config_t &config, pix_fmt_e pix_fmt, const video::sunshine_colorspace_t &colorspace) {
return nullptr;
}

Expand Down
4 changes: 2 additions & 2 deletions src/platform/windows/display.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,10 +217,10 @@ namespace platf::dxgi {
init(const ::video::config_t &config, const std::string &display_name);

std::unique_ptr<avcodec_encoding_stream_t>
make_avcodec_encoding_stream(pix_fmt_e pix_fmt) override;
make_avcodec_encoding_stream(pix_fmt_e pix_fmt, const ::video::sunshine_colorspace_t &colorspace) override;

std::unique_ptr<nvenc_encoding_stream_t>
make_nvenc_encoding_stream(const ::video::config_t &config, pix_fmt_e pix_fmt) override;
make_nvenc_encoding_stream(const ::video::config_t &config, pix_fmt_e pix_fmt, const ::video::sunshine_colorspace_t &colorspace) override;

sampler_state_t sampler_linear;

Expand Down
121 changes: 34 additions & 87 deletions src/platform/windows/display_vram.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -416,39 +416,6 @@ namespace platf::dxgi {
return 0;
}

void
set_colorspace(std::uint32_t colorspace, std::uint32_t color_range) {
switch (colorspace) {
case 5: // SWS_CS_SMPTE170M
color_p = &::video::colors[0];
break;
case 1: // SWS_CS_ITU709
color_p = &::video::colors[2];
break;
case 9: // SWS_CS_BT2020
color_p = &::video::colors[4];
break;
default:
BOOST_LOG(warning) << "Colorspace: ["sv << colorspace << "] not yet supported: switching to default"sv;
color_p = &::video::colors[0];
};

if (color_range > 1) {
// Full range
++color_p;
}

auto color_matrix = make_buffer(device.get(), *color_p);
if (!color_matrix) {
BOOST_LOG(warning) << "Failed to create color matrix"sv;
return;
}

device_ctx->VSSetConstantBuffers(0, 1, &info_scene);
device_ctx->PSSetConstantBuffers(0, 1, &color_matrix);
this->color_matrix = std::move(color_matrix);
}

int
init_output(ID3D11Texture2D *frame_texture, int width, int height) {
// The underlying frame pool owns the texture, so we must reference it for ourselves
Expand Down Expand Up @@ -510,7 +477,7 @@ namespace platf::dxgi {
}

int
init(std::shared_ptr<platf::display_t> display, adapter_t::pointer adapter_p, pix_fmt_e pix_fmt) {
init(std::shared_ptr<platf::display_t> display, adapter_t::pointer adapter_p, pix_fmt_e pix_fmt, const ::video::sunshine_colorspace_t &colorspace) {
D3D_FEATURE_LEVEL featureLevels[] {
D3D_FEATURE_LEVEL_11_1,
D3D_FEATURE_LEVEL_11_0,
Expand Down Expand Up @@ -605,7 +572,13 @@ namespace platf::dxgi {
return -1;
}

color_matrix = make_buffer(device.get(), ::video::colors[0]);
auto default_color_vectors = ::video::color_vectors_from_colorspace(::video::colorspace_e::rec601, false);
if (!default_color_vectors) {
BOOST_LOG(error) << "Missing color vectors for Rec. 601"sv;
return -1;
}

color_matrix = make_buffer(device.get(), *default_color_vectors);
if (!color_matrix) {
BOOST_LOG(error) << "Failed to create color matrix buffer"sv;
return -1;
Expand Down Expand Up @@ -650,6 +623,23 @@ namespace platf::dxgi {
device_ctx->PSSetSamplers(0, 1, &sampler_linear);
device_ctx->IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLESTRIP);

auto color_vectors = ::video::color_vectors_from_colorspace(colorspace);

if (!color_vectors) {
BOOST_LOG(error) << "No vector data for colorspace"sv;
return -1;
}

auto color_matrix = make_buffer(device.get(), *color_vectors);
if (!color_matrix) {
BOOST_LOG(warning) << "Failed to create color matrix"sv;
return -1;
}

device_ctx->VSSetConstantBuffers(0, 1, &info_scene);
device_ctx->PSSetConstantBuffers(0, 1, &color_matrix);
this->color_matrix = std::move(color_matrix);

return 0;
}

Expand Down Expand Up @@ -762,8 +752,8 @@ namespace platf::dxgi {
class d3d_avcodec_encoding_stream_t: public avcodec_encoding_stream_t {
public:
int
init(std::shared_ptr<platf::display_t> display, adapter_t::pointer adapter_p, pix_fmt_e pix_fmt) {
int result = base.init(display, adapter_p, pix_fmt);
init(std::shared_ptr<platf::display_t> display, adapter_t::pointer adapter_p, pix_fmt_e pix_fmt, const ::video::sunshine_colorspace_t &colorspace) {
int result = base.init(display, adapter_p, pix_fmt, colorspace);
data = base.device.get();
return result;
}
Expand All @@ -773,11 +763,6 @@ namespace platf::dxgi {
return base.convert(img_base);
}

void
set_colorspace(std::uint32_t colorspace, std::uint32_t color_range) override {
base.set_colorspace(colorspace, color_range);
}

void
init_hwframes(AVHWFramesContext *frames) override {
// We may be called with a QSV or D3D11VA context
Expand Down Expand Up @@ -859,48 +844,18 @@ namespace platf::dxgi {
class d3d_nvenc_encoding_stream_t: public nvenc_encoding_stream_t {
public:
bool
init(std::shared_ptr<platf::display_t> display, adapter_t::pointer adapter_p, const ::video::config_t &client_config, pix_fmt_e pix_fmt) {
init(std::shared_ptr<platf::display_t> display, adapter_t::pointer adapter_p, const ::video::config_t &client_config, pix_fmt_e pix_fmt, const ::video::sunshine_colorspace_t &colorspace) {
auto buffer_format = nvenc::nvenc_format_from_sunshine_format(pix_fmt);
if (buffer_format == NV_ENC_BUFFER_FORMAT_UNDEFINED) {
BOOST_LOG(error) << "Unexpected pixel format for NvENC ["sv << from_pix_fmt(pix_fmt) << ']';
return false;
}

if (base.init(display, adapter_p, pix_fmt)) return false;

// TODO: add sunshine colorspace/primaries/tranfer/matrix constants and translate those to avcodec, nvenc, whatever
{
int color_range = (client_config.encoderCscMode & 0x1) ? 2 : 1;
int color_space;
if (client_config.dynamicRange && display->is_hdr()) {
// When HDR is active, that overrides the colorspace the client requested
color_space = 9;
}
else {
switch (client_config.encoderCscMode >> 1) {
case 0:
default:
// BT.601
color_space = 5;
break;

case 1:
// BT.709
color_space = 1;
break;

case 2:
// BT.2020
color_space = 9;
break;
}
}
set_colorspace(color_space, color_range);
}
if (base.init(display, adapter_p, pix_fmt, colorspace)) return false;

nvenc = std::make_unique<nvenc::nvenc_d3d11>(base.device.get(), client_config.width, client_config.height, buffer_format);

auto nvenc_colorspace = nvenc::nvenc_colorspace_from_rtsp_csc(client_config.encoderCscMode, client_config.dynamicRange && display->is_hdr());
auto nvenc_colorspace = nvenc::nvenc_colorspace_from_sunshine_colorspace(colorspace);

nvenc::nvenc_config nvenc_config;

Expand All @@ -921,11 +876,6 @@ namespace platf::dxgi {
return base.convert(img_base);
}

void
set_colorspace(std::uint32_t colorspace, std::uint32_t color_range) override {
base.set_colorspace(colorspace, color_range);
}

void
request_idr_frame() override {
force_idr = true;
Expand Down Expand Up @@ -1585,7 +1535,7 @@ namespace platf::dxgi {
}

std::unique_ptr<avcodec_encoding_stream_t>
display_vram_t::make_avcodec_encoding_stream(pix_fmt_e pix_fmt) {
display_vram_t::make_avcodec_encoding_stream(pix_fmt_e pix_fmt, const ::video::sunshine_colorspace_t &colorspace) {
if (pix_fmt != platf::pix_fmt_e::nv12 && pix_fmt != platf::pix_fmt_e::p010) {
BOOST_LOG(error) << "display_vram_t doesn't support pixel format ["sv << from_pix_fmt(pix_fmt) << ']';

Expand All @@ -1594,10 +1544,7 @@ namespace platf::dxgi {

auto hwdevice = std::make_unique<d3d_avcodec_encoding_stream_t>();

auto ret = hwdevice->init(
shared_from_this(),
adapter.get(),
pix_fmt);
auto ret = hwdevice->init(shared_from_this(), adapter.get(), pix_fmt, colorspace);

if (ret) {
return nullptr;
Expand All @@ -1607,9 +1554,9 @@ namespace platf::dxgi {
}

std::unique_ptr<nvenc_encoding_stream_t>
display_vram_t::make_nvenc_encoding_stream(const ::video::config_t &config, pix_fmt_e pix_fmt) {
display_vram_t::make_nvenc_encoding_stream(const ::video::config_t &config, pix_fmt_e pix_fmt, const ::video::sunshine_colorspace_t &colorspace) {
auto encoding_stream = std::make_unique<d3d_nvenc_encoding_stream_t>();
if (!encoding_stream->init(shared_from_this(), adapter.get(), config, pix_fmt)) {
if (!encoding_stream->init(shared_from_this(), adapter.get(), config, pix_fmt, colorspace)) {
return nullptr;
}
return encoding_stream;
Expand Down
Loading

0 comments on commit 6836c6e

Please sign in to comment.