Check encoder caps

LizardByte · Jul 11, 2023 · 3b82773 · 3b82773
1 parent a2f34ab
commit 3b82773
Show file tree

Hide file tree

Showing 2 changed files with 72 additions and 16 deletions.
diff --git a/src/nvenc/nvenc_base.cpp b/src/nvenc/nvenc_base.cpp
@@ -86,22 +86,19 @@ namespace nvenc {
       return false;
     }
 
-    width = client_config.width;
-    height = client_config.height;
+    uint32_t encode_guid_count = 0;
+    if (nvenc->nvEncGetEncodeGUIDCount(encoder, &encode_guid_count) != NV_ENC_SUCCESS) {
+      BOOST_LOG(error) << "NvEncGetEncodeGUIDCount failed: " << nvenc->nvEncGetLastErrorString(encoder);
+      return false;
+    };
 
-    NV_ENC_INITIALIZE_PARAMS init_params = { NV_ENC_INITIALIZE_PARAMS_VER };
-    init_params.maxEncodeWidth = max_width;
-    init_params.maxEncodeHeight = max_height;
-    init_params.presetGUID = quality_preset_guid_from_number(config.quality_preset);
-    init_params.tuningInfo = NV_ENC_TUNING_INFO_ULTRA_LOW_LATENCY;
-    init_params.enablePTD = 1;
+    std::vector<GUID> encode_guids(encode_guid_count);
+    if (nvenc->nvEncGetEncodeGUIDs(encoder, encode_guids.data(), encode_guids.size(), &encode_guid_count) != NV_ENC_SUCCESS) {
+      BOOST_LOG(error) << "NvEncGetEncodeGUIDs failed: " << nvenc->nvEncGetLastErrorString(encoder);
+      return false;
+    }
 
-    init_params.encodeWidth = width;
-    init_params.darWidth = width;
-    init_params.encodeHeight = height;
-    init_params.darHeight = height;
-    init_params.frameRateNum = client_config.framerate;
-    init_params.frameRateDen = 1;
+    NV_ENC_INITIALIZE_PARAMS init_params = { NV_ENC_INITIALIZE_PARAMS_VER };
 
     switch (client_config.videoFormat) {
       case 0:
@@ -115,6 +112,52 @@ namespace nvenc {
         break;
     }
 
+    if (std::find(encode_guids.begin(), encode_guids.end(), init_params.encodeGUID) == encode_guids.end()) {
+      // Video format is not supported by the encoder
+      return false;
+    }
+
+    auto get_encoder_cap = [&](NV_ENC_CAPS cap) {
+      NV_ENC_CAPS_PARAM param = { NV_ENC_CAPS_PARAM_VER, cap };
+      int value = 0;
+      nvenc->nvEncGetEncodeCaps(encoder, init_params.encodeGUID, &param, &value);
+      return value;
+    };
+
+    if (max_width > get_encoder_cap(NV_ENC_CAPS_WIDTH_MAX) || max_height > get_encoder_cap(NV_ENC_CAPS_HEIGHT_MAX)) {
+      // Encoder doesn't support requested dimensions
+      return false;
+    }
+
+    if (buffer_format == NV_ENC_BUFFER_FORMAT_YUV420_10BIT || buffer_format == NV_ENC_BUFFER_FORMAT_YUV444_10BIT) {
+      if (!get_encoder_cap(NV_ENC_CAPS_SUPPORT_10BIT_ENCODE)) {
+        // Encoder doesn't support 10-bit encode
+        return false;
+      }
+    }
+
+    if (buffer_format == NV_ENC_BUFFER_FORMAT_YUV444 || buffer_format == NV_ENC_BUFFER_FORMAT_YUV444_10BIT) {
+      if (!get_encoder_cap(NV_ENC_CAPS_SUPPORT_YUV444_ENCODE)) {
+        // Encoder doesn't support yuv444 encode
+        return false;
+      }
+    }
+
+    supporting_ref_frame_invalidation = get_encoder_cap(NV_ENC_CAPS_SUPPORT_REF_PIC_INVALIDATION);
+
+    init_params.maxEncodeWidth = max_width;
+    init_params.maxEncodeHeight = max_height;
+    init_params.presetGUID = quality_preset_guid_from_number(config.quality_preset);
+    init_params.tuningInfo = NV_ENC_TUNING_INFO_ULTRA_LOW_LATENCY;
+    init_params.enablePTD = 1;
+
+    init_params.encodeWidth = client_config.width;
+    init_params.darWidth = client_config.width;
+    init_params.encodeHeight = client_config.height;
+    init_params.darHeight = client_config.height;
+    init_params.frameRateNum = client_config.framerate;
+    init_params.frameRateDen = 1;
+
     NV_ENC_PRESET_CONFIG preset_config = { NV_ENC_PRESET_CONFIG_VER, { NV_ENC_CONFIG_VER } };
     if (nvenc->nvEncGetEncodePresetConfigEx(encoder, init_params.encodeGUID, init_params.presetGUID, init_params.tuningInfo, &preset_config) != NV_ENC_SUCCESS) {
       BOOST_LOG(error) << "NvEncGetEncodePresetConfigEx failed: " << nvenc->nvEncGetLastErrorString(encoder);
@@ -135,7 +178,10 @@ namespace nvenc {
                                     config.multipass == multipass_e::two_pass_quarter_res ? NV_ENC_TWO_PASS_QUARTER_RESOLUTION :
                                                                                             NV_ENC_MULTI_PASS_DISABLED;
     enc_config.rcParams.averageBitRate = client_config.bitrate * 1000;
-    enc_config.rcParams.vbvBufferSize = client_config.bitrate * 1000 / client_config.framerate;
+
+    if (get_encoder_cap(NV_ENC_CAPS_SUPPORT_CUSTOM_VBV_BUF_SIZE)) {
+      enc_config.rcParams.vbvBufferSize = client_config.bitrate * 1000 / client_config.framerate;
+    }
 
     auto set_common_format_config = [&config, &client_config, this](auto &format_config) {
       format_config.repeatSPSPPS = 1;
@@ -163,12 +209,17 @@ namespace nvenc {
         // H.264
         auto &format_config = enc_config.encodeCodecConfig.h264Config;
         set_common_format_config(format_config);
+        format_config.entropyCodingMode = get_encoder_cap(NV_ENC_CAPS_SUPPORT_CABAC) ? NV_ENC_H264_ENTROPY_CODING_MODE_CABAC : NV_ENC_H264_ENTROPY_CODING_MODE_CAVLC;
         if (client_config.numRefFrames > 0) {
           format_config.maxNumRefFrames = client_config.numRefFrames;
         }
         else {
           format_config.maxNumRefFrames = 5;
         }
+        if (format_config.maxNumRefFrames > 0 && !get_encoder_cap(NV_ENC_CAPS_SUPPORT_MULTIPLE_REF_FRAMES)) {
+          format_config.maxNumRefFrames = 1;
+          supporting_ref_frame_invalidation = false;
+        }
         format_config.numRefL0 = NV_ENC_NUM_REF_FRAMES_1;
         fill_vui(format_config.h264VUIParameters);
         break;
@@ -187,6 +238,10 @@ namespace nvenc {
         else {
           format_config.maxNumRefFramesInDPB = 5;
         }
+        if (format_config.maxNumRefFramesInDPB > 0 && !get_encoder_cap(NV_ENC_CAPS_SUPPORT_MULTIPLE_REF_FRAMES)) {
+          format_config.maxNumRefFramesInDPB = 1;
+          supporting_ref_frame_invalidation = false;
+        }
         format_config.numRefL0 = NV_ENC_NUM_REF_FRAMES_1;
         fill_vui(format_config.hevcVUIParameters);
         break;
@@ -281,7 +336,7 @@ namespace nvenc {
 
   bool
   nvenc_base::invalidate_ref_frames(uint64_t first_frame, uint64_t last_frame) {
-    if (!encoder) return false;
+    if (!encoder || !supporting_ref_frame_invalidation) return false;
 
     if (last_frame < first_frame || last_encoded_frame_index < first_frame || last_encoded_frame_index > first_frame + 100) {
       BOOST_LOG(error) << "invalidate_ref_frames " << first_frame << "-" << last_frame << " invalid range (current frame " << last_encoded_frame_index << ")";

diff --git a/src/nvenc/nvenc_base.h b/src/nvenc/nvenc_base.h
@@ -56,6 +56,7 @@ namespace nvenc {
 
     uint64_t last_encoded_frame_index = 0;
 
+    bool supporting_ref_frame_invalidation = true;
     bool ref_frame_invalidation_requested = false;
     std::pair<uint64_t, uint64_t> last_ref_frame_invalidation_range;
   };