Initial implementation of yuv444in420 encoding

src/nvenc/nvenc_base.cpp

@@ -102,6 +102,7 @@ namespace nvenc {
 
     encoder_params.width = client_config.width;
     encoder_params.height = client_config.height;
+    if (client_config.chromaSamplingType == 2) encoder_params.height *= 2;  // YUV 4:4:4 recombined into YUV 4:2:0
     encoder_params.buffer_format = buffer_format;
     encoder_params.rfi = true;
 
@@ -286,8 +287,8 @@ namespace nvenc {
       vui_config.transferCharacteristics = colorspace.tranfer_function;
       vui_config.colourMatrix = colorspace.matrix;
       vui_config.chromaSampleLocationFlag = buffer_is_yuv444() ? 0 : 1;
-      vui_config.chromaSampleLocationTop = 0;
-      vui_config.chromaSampleLocationBot = 0;
+      vui_config.chromaSampleLocationTop = (!buffer_is_yuv444() && client_config.chromaSamplingType == 2) ? 1 : 0;
+      vui_config.chromaSampleLocationBot = (!buffer_is_yuv444() && client_config.chromaSamplingType == 2) ? 1 : 0;
     };
 
     switch (client_config.videoFormat) {
@@ -338,7 +339,9 @@ namespace nvenc {
         format_config.transferCharacteristics = colorspace.tranfer_function;
         format_config.matrixCoefficients = colorspace.matrix;
         format_config.colorRange = colorspace.full_range;
-        format_config.chromaSamplePosition = buffer_is_yuv444() ? 0 : 1;
+        format_config.chromaSamplePosition = buffer_is_yuv444()                    ? 0 :
+                                             client_config.chromaSamplingType == 2 ? 3 :
+                                                                                     1;
         set_ref_frames(format_config.maxNumRefFramesInDPB, format_config.numFwdRefs, 8);
         set_minqp_if_enabled(config.min_qp_av1);
 

src/platform/common.h

@@ -477,12 +477,12 @@ namespace platf {
     dummy_img(img_t *img) = 0;
 
     virtual std::unique_ptr<avcodec_encode_device_t>
-    make_avcodec_encode_device(pix_fmt_e pix_fmt) {
+    make_avcodec_encode_device(pix_fmt_e pix_fmt, bool yuv444in420) {
       return nullptr;
     }
 
     virtual std::unique_ptr<nvenc_encode_device_t>
-    make_nvenc_encode_device(pix_fmt_e pix_fmt) {
+    make_nvenc_encode_device(pix_fmt_e pix_fmt, bool yuv444in420) {
       return nullptr;
     }
 

src/platform/windows/display.h

@@ -275,7 +275,7 @@ namespace platf::dxgi {
     get_supported_capture_formats() override;
 
     std::unique_ptr<avcodec_encode_device_t>
-    make_avcodec_encode_device(pix_fmt_e pix_fmt) override;
+    make_avcodec_encode_device(pix_fmt_e pix_fmt, bool yuv444in420) override;
 
     D3D11_MAPPED_SUBRESOURCE img_info;
     texture2d_t texture;
@@ -299,10 +299,10 @@ namespace platf::dxgi {
     is_codec_supported(std::string_view name, const ::video::config_t &config) override;
 
     std::unique_ptr<avcodec_encode_device_t>
-    make_avcodec_encode_device(pix_fmt_e pix_fmt) override;
+    make_avcodec_encode_device(pix_fmt_e pix_fmt, bool yuv444in420) override;
 
     std::unique_ptr<nvenc_encode_device_t>
-    make_nvenc_encode_device(pix_fmt_e pix_fmt) override;
+    make_nvenc_encode_device(pix_fmt_e pix_fmt, bool yuv444in420) override;
 
     std::atomic<uint32_t> next_image_id;
   };

src/platform/windows/display_ram.cpp

@@ -398,7 +398,11 @@ namespace platf::dxgi {
   }
 
   std::unique_ptr<avcodec_encode_device_t>
-  display_ram_t::make_avcodec_encode_device(pix_fmt_e pix_fmt) {
+  display_ram_t::make_avcodec_encode_device(pix_fmt_e pix_fmt, bool yuv444in420) {
+    if (yuv444in420) {
+      BOOST_LOG(error) << "Recombined YUV 4:4:4 is not supported";
+      return nullptr;
+    }
     return std::make_unique<avcodec_encode_device_t>();
   }
 

src/rtsp.cpp

@@ -1020,7 +1020,8 @@ namespace rtsp_stream {
       config.monitor.encoderCscMode = util::from_view(args.at("x-nv-video[0].encoderCscMode"sv));
       config.monitor.videoFormat = util::from_view(args.at("x-nv-vqos[0].bitStreamFormat"sv));
       config.monitor.dynamicRange = util::from_view(args.at("x-nv-video[0].dynamicRangeMode"sv));
-      config.monitor.chromaSamplingType = util::from_view(args.at("x-ss-video[0].chromaSamplingType"sv));
+      // config.monitor.chromaSamplingType = util::from_view(args.at("x-ss-video[0].chromaSamplingType"sv));
+      config.monitor.chromaSamplingType = 2;
 
       configuredBitrateKbps = util::from_view(args.at("x-ml-video.configuredBitrateKbps"sv));
     }

src/video.cpp

@@ -1417,11 +1417,18 @@ namespace video {
     }
 
     auto colorspace = encode_device->colorspace;
-    auto sw_fmt = (colorspace.bit_depth == 8 && config.chromaSamplingType == 0)  ? platform_formats->avcodec_pix_fmt_8bit :
-                  (colorspace.bit_depth == 8 && config.chromaSamplingType == 1)  ? platform_formats->avcodec_pix_fmt_yuv444_8bit :
-                  (colorspace.bit_depth == 10 && config.chromaSamplingType == 0) ? platform_formats->avcodec_pix_fmt_10bit :
-                  (colorspace.bit_depth == 10 && config.chromaSamplingType == 1) ? platform_formats->avcodec_pix_fmt_yuv444_10bit :
-                                                                                   AV_PIX_FMT_NONE;
+
+    AVPixelFormat sw_fmt = AV_PIX_FMT_NONE;
+    if (colorspace.bit_depth == 8) {
+      sw_fmt = (config.chromaSamplingType == 1) ?
+                 platform_formats->avcodec_pix_fmt_yuv444_8bit :
+                 platform_formats->avcodec_pix_fmt_8bit;
+    }
+    else if (colorspace.bit_depth == 10) {
+      sw_fmt = (config.chromaSamplingType == 1) ?
+                 platform_formats->avcodec_pix_fmt_yuv444_10bit :
+                 platform_formats->avcodec_pix_fmt_10bit;
+    }
 
     // Allow up to 1 retry to apply the set of fallback options.
     //
@@ -1669,7 +1676,7 @@ namespace video {
     frame->color_primaries = ctx->color_primaries;
     frame->color_trc = ctx->color_trc;
     frame->colorspace = ctx->colorspace;
-    frame->chroma_location = ctx->chroma_sample_location;
+    frame->chroma_location = (config.chromaSamplingType == 2) ? AVCHROMA_LOC_CENTER : ctx->chroma_sample_location;
 
     // Attach HDR metadata to the AVFrame
     if (colorspace_is_hdr(colorspace)) {
@@ -1921,10 +1928,10 @@ namespace video {
     }
 
     if (dynamic_cast<const encoder_platform_formats_avcodec *>(encoder.platform_formats.get())) {
-      result = disp.make_avcodec_encode_device(pix_fmt);
+      result = disp.make_avcodec_encode_device(pix_fmt, config.chromaSamplingType == 2);
     }
     else if (dynamic_cast<const encoder_platform_formats_nvenc *>(encoder.platform_formats.get())) {
-      result = disp.make_nvenc_encode_device(pix_fmt);
+      result = disp.make_nvenc_encode_device(pix_fmt, config.chromaSamplingType == 2);
     }
 
     if (result) {

src/video.h

@@ -327,7 +327,7 @@ namespace video {
        HDR encoding activates when color depth is higher than 8-bit and the display which is being captured is operating in HDR mode */
     int dynamicRange;
 
-    int chromaSamplingType;  // 0 - 4:2:0, 1 - 4:4:4
+    int chromaSamplingType;  // 0 - 4:2:0, 1 - 4:4:4, 2 - 4:4:4 recombined into 4:2:0
   };
 
   extern int active_hevc_mode;

src_assets/windows/assets/shaders/directx/convert_yuv444in420_nv12_uv_ps.hlsl

@@ -0,0 +1,3 @@
+#include "include/convert_base.hlsl"
+
+#include "include/convert_yuv444in420_ps_base.hlsl"

src_assets/windows/assets/shaders/directx/convert_yuv444in420_nv12_uv_ps_linear.hlsl

@@ -0,0 +1,3 @@
+#include "include/convert_linear_base.hlsl"
+
+#include "include/convert_yuv444in420_ps_base.hlsl"

src_assets/windows/assets/shaders/directx/convert_yuv444in420_nv12_y_ps.hlsl

@@ -0,0 +1,4 @@
+#include "include/convert_base.hlsl"
+
+#define PLANAR_VIEWPORTS
+#include "include/convert_yuv444in420_ps_base.hlsl"

src_assets/windows/assets/shaders/directx/convert_yuv444in420_nv12_y_ps_linear.hlsl

@@ -0,0 +1,4 @@
+#include "include/convert_linear_base.hlsl"
+
+#define PLANAR_VIEWPORTS
+#include "include/convert_yuv444in420_ps_base.hlsl"

src_assets/windows/assets/shaders/directx/convert_yuv444in420_p010_uv_ps.hlsl

@@ -0,0 +1,4 @@
+#include "include/convert_base.hlsl"
+
+#define P010
+#include "include/convert_yuv444in420_ps_base.hlsl"

src_assets/windows/assets/shaders/directx/convert_yuv444in420_p010_uv_ps_linear.hlsl

@@ -0,0 +1,4 @@
+#include "include/convert_linear_base.hlsl"
+
+#define P010
+#include "include/convert_yuv444in420_ps_base.hlsl"

src_assets/windows/assets/shaders/directx/convert_yuv444in420_p010_uv_ps_perceptual_quantizer.hlsl

@@ -0,0 +1,4 @@
+#include "include/convert_perceptual_quantizer_base.hlsl"
+
+#define P010
+#include "include/convert_yuv444in420_ps_base.hlsl"

src_assets/windows/assets/shaders/directx/convert_yuv444in420_p010_y_ps.hlsl

@@ -0,0 +1,5 @@
+#include "include/convert_base.hlsl"
+
+#define PLANAR_VIEWPORTS
+#define P010
+#include "include/convert_yuv444in420_ps_base.hlsl"

src_assets/windows/assets/shaders/directx/convert_yuv444in420_p010_y_ps_linear.hlsl

@@ -0,0 +1,5 @@
+#include "include/convert_linear_base.hlsl"
+
+#define PLANAR_VIEWPORTS
+#define P010
+#include "include/convert_yuv444in420_ps_base.hlsl"

src_assets/windows/assets/shaders/directx/convert_yuv444in420_p010_y_ps_perceptual_quantizer.hlsl

@@ -0,0 +1,5 @@
+#include "include/convert_perceptual_quantizer_base.hlsl"
+
+#define PLANAR_VIEWPORTS
+#define P010
+#include "include/convert_yuv444in420_ps_base.hlsl"

src_assets/windows/assets/shaders/directx/convert_yuv444in420_uv_vs.hlsl

@@ -0,0 +1 @@
+#include "include/convert_yuv444in420_vs_base.hlsl"

src_assets/windows/assets/shaders/directx/convert_yuv444in420_y_vs.hlsl

@@ -0,0 +1,2 @@
+#define PLANAR_VIEWPORTS
+#include "include/convert_yuv444in420_vs_base.hlsl"

src_assets/windows/assets/shaders/directx/include/convert_yuv444in420_ps_base.hlsl

@@ -0,0 +1,51 @@
+Texture2D image : register(t0);
+SamplerState def_sampler : register(s0);
+
+#ifndef PLANAR_VIEWPORTS
+cbuffer color_matrix_cbuffer : register(b0) {
+    float4 color_vec_y;
+    float4 color_vec_u;
+    float4 color_vec_v;
+    float2 range_y;
+    float2 range_uv;
+};
+#endif
+
+#include "include/base_vs_types.hlsl"
+
+#ifdef PLANAR_VIEWPORTS
+uint main_ps(vertex_t input) : SV_Target
+#else
+uint2 main_ps(vertex_t input) : SV_Target
+#endif
+{
+    //       Y         U     V
+    //       +-------+ +---+ +---+
+    //       |       | |   | |   |
+    // VP0-> |   Y   | |UR | |VR |
+    //       |       | |   | |   |
+    //       +---+---+ +---+ +---+
+    //       |   |   |
+    // VP1-> |UL |VL | <-VP2
+    //       |   |   |
+    //       +---+---+
+
+    float3 rgb = CONVERT_FUNCTION(image.Sample(def_sampler, input.tex_coord, 0).rgb);
+
+#ifdef PLANAR_VIEWPORTS
+    float y = dot(input.color_vec.xyz, rgb) + input.color_vec.w;
+#ifdef P010
+    return uint(y) << 6;
+#else
+    return uint(y);
+#endif
+#else
+    float u = dot(color_vec_u.xyz, rgb) + color_vec_u.w;
+    float v = dot(color_vec_v.xyz, rgb) + color_vec_v.w;
+#ifdef P010
+    return uint2(u, v) << 6;
+#else
+    return uint2(u, v);
+#endif
+#endif
+}

src_assets/windows/assets/shaders/directx/include/convert_yuv444in420_vs_base.hlsl

@@ -0,0 +1,125 @@
+cbuffer rotate_texture_steps_cbuffer : register(b1) {
+    int rotate_texture_steps;
+};
+
+#ifdef PLANAR_VIEWPORTS
+cbuffer color_matrix_cbuffer : register(b3) {
+    float4 color_vec_y;
+    float4 color_vec_u;
+    float4 color_vec_v;
+    float2 range_y;
+    float2 range_uv;
+};
+#endif
+
+#include "include/base_vs_types.hlsl"
+
+vertex_t main_vs(uint vertex_id : SV_VertexID)
+{
+    vertex_t output;
+    float2 tex_coord;
+
+    // id=1 -> +
+    //         |\
+    //         | \
+    //         |  \
+    //         +---+
+    //         |   |\
+    // id=0 -> +---+-+ <- id=3
+
+    //       Y         U     V
+    //       +-------+ +---+ +---+
+    //       |       | |   | |   |
+    // VP0-> |   Y   | |UR | |VR |
+    //       |       | |   | |   |
+    //       +---+---+ +---+ +---+
+    //       |   |   |
+    // VP1-> |UL |VL | <-VP2
+    //       |   |   |
+    //       +---+---+
+
+#ifdef PLANAR_VIEWPORTS
+    switch (vertex_id) {
+        case 0:
+            output.viewpoint_pos = float4(-1, -1, 0, 1);
+            tex_coord = float2(0, 1);
+            output.viewport = 0;
+            output.color_vec = color_vec_y;
+            break;
+        case 1:
+            output.viewpoint_pos = float4(-1, 3, 0, 1);
+            tex_coord = float2(0, -1);
+            output.viewport = 0;
+            output.color_vec = color_vec_y;
+            break;
+        case 2:
+            output.viewpoint_pos = float4(3, -1, 0, 1);
+            tex_coord = float2(2, 1);
+            output.viewport = 0;
+            output.color_vec = color_vec_y;
+            break;
+
+        case 3:
+            output.viewpoint_pos = float4(-1, -1, 0, 1);
+            tex_coord = float2(0, 1);
+            output.viewport = 1;
+            output.color_vec = color_vec_u;
+            break;
+        case 4:
+            output.viewpoint_pos = float4(-1, 3, 0, 1);
+            tex_coord = float2(0, -1);
+            output.viewport = 1;
+            output.color_vec = color_vec_u;
+            break;
+        case 5:
+            output.viewpoint_pos = float4(7, -1, 0, 1);
+            tex_coord = float2(2, 1);
+            output.viewport = 1;
+            output.color_vec = color_vec_u;
+            break;
+
+        case 6:
+            output.viewpoint_pos = float4(-1, -1, 0, 1);
+            tex_coord = float2(0, 1);
+            output.viewport = 2;
+            output.color_vec = color_vec_v;
+            break;
+        case 7:
+            output.viewpoint_pos = float4(-1, 3, 0, 1);
+            tex_coord = float2(0, -1);
+            output.viewport = 2;
+            output.color_vec = color_vec_v;
+            break;
+        case 8:
+            output.viewpoint_pos = float4(7, -1, 0, 1);
+            tex_coord = float2(2, 1);
+            output.viewport = 2;
+            output.color_vec = color_vec_v;
+            break;
+    }
+#else
+    if (vertex_id == 0) {
+        output.viewpoint_pos = float4(-3, -1, 0, 1);
+        tex_coord = float2(0, 1);
+    }
+    else if (vertex_id == 1) {
+        output.viewpoint_pos = float4(-3, 5, 0, 1);
+        tex_coord = float2(0, -2);
+    }
+    else {
+        output.viewpoint_pos = float4(5, -1, 0, 1);
+        tex_coord = float2(2, 1);
+    }
+#endif
+
+    if (rotate_texture_steps != 0) {
+        float rotation_radians = radians(90 * rotate_texture_steps);
+        float2x2 rotation_matrix = { cos(rotation_radians), -sin(rotation_radians),
+                                     sin(rotation_radians), cos(rotation_radians) };
+        float2 rotation_center = { 0.5, 0.5 };
+        tex_coord = round(rotation_center + mul(rotation_matrix, tex_coord - rotation_center));
+    }
+    output.tex_coord = tex_coord;
+
+    return output;
+}