Encoder probing performance improvements

src/platform/common.h

@@ -558,6 +558,13 @@ namespace platf {
   std::vector<std::string>
   display_names(mem_type_e hwdevice_type);
 
+  /**
+   * @brief Returns if GPUs/drivers have changed since the last call to this function.
+   * @return `true` if a change has occurred or if it is unknown whether a change occurred.
+   */
+  bool
+  needs_encoder_reenumeration();
+
   boost::process::child
   run_command(bool elevated, bool interactive, const std::string &cmd, boost::filesystem::path &working_dir, const boost::process::environment &env, FILE *file, std::error_code &ec, boost::process::group *group);
 

src/platform/linux/misc.cpp

@@ -773,6 +773,16 @@ namespace platf {
     return {};
   }
 
+  /**
+   * @brief Returns if GPUs/drivers have changed since the last call to this function.
+   * @return `true` if a change has occurred or if it is unknown whether a change occurred.
+   */
+  bool
+  needs_encoder_reenumeration() {
+    // We don't track GPU state, so we will always reenumerate. Fortunately, it is fast on Linux.
+    return true;
+  }
+
   std::shared_ptr<display_t>
   display(mem_type_e hwdevice_type, const std::string &display_name, const video::config_t &config) {
 #ifdef SUNSHINE_BUILD_CUDA

src/platform/macos/display.mm

@@ -181,4 +181,14 @@
 
     return display_names;
   }
+
+  /**
+   * @brief Returns if GPUs/drivers have changed since the last call to this function.
+   * @return `true` if a change has occurred or if it is unknown whether a change occurred.
+   */
+  bool
+  needs_encoder_reenumeration() {
+    // We don't track GPU state, so we will always reenumerate. Fortunately, it is fast on macOS.
+    return true;
+  }
 }  // namespace platf

src/platform/windows/display_base.cpp

@@ -1124,4 +1124,35 @@ namespace platf {
     return display_names;
   }
 
+  /**
+   * @brief Returns if GPUs/drivers have changed since the last call to this function.
+   * @return `true` if a change has occurred or if it is unknown whether a change occurred.
+   */
+  bool
+  needs_encoder_reenumeration() {
+    // Serialize access to the static DXGI factory
+    static std::mutex reenumeration_state_lock;
+    auto lg = std::lock_guard(reenumeration_state_lock);
+
+    // Keep a reference to the DXGI factory, which will keep track of changes internally.
+    static dxgi::factory1_t factory;
+    if (!factory || !factory->IsCurrent()) {
+      factory.reset();
+
+      auto status = CreateDXGIFactory1(IID_IDXGIFactory1, (void **) &factory);
+      if (FAILED(status)) {
+        BOOST_LOG(error) << "Failed to create DXGIFactory1 [0x"sv << util::hex(status).to_string_view() << ']';
+        factory.release();
+      }
+
+      // Always request reenumeration on the first streaming session just to ensure we
+      // can deal with any initialization races that may occur when the system is booting.
+      BOOST_LOG(info) << "Encoder reenumeration is required"sv;
+      return true;
+    }
+    else {
+      // The DXGI factory from last time is still current, so no encoder changes have occurred.
+      return false;
+    }
+  }
 }  // namespace platf

src/video.cpp

@@ -276,14 +276,15 @@ namespace video {
 
   enum flag_e : uint32_t {
     DEFAULT = 0,
-    PARALLEL_ENCODING = 1 << 1,
+    PARALLEL_ENCODING = 1 << 1,  // Capture and encoding can run concurrently on separate threads
     H264_ONLY = 1 << 2,  // When HEVC is too heavy
     LIMITED_GOP_SIZE = 1 << 3,  // Some encoders don't like it when you have an infinite GOP_SIZE. *cough* VAAPI *cough*
     SINGLE_SLICE_ONLY = 1 << 4,  // Never use multiple slices <-- Older intel iGPU's ruin it for everyone else :P
     CBR_WITH_VBR = 1 << 5,  // Use a VBR rate control mode to simulate CBR
     RELAXED_COMPLIANCE = 1 << 6,  // Use FF_COMPLIANCE_UNOFFICIAL compliance mode
     NO_RC_BUF_LIMIT = 1 << 7,  // Don't set rc_buffer_size
     REF_FRAMES_INVALIDATION = 1 << 8,  // Support reference frames invalidation
+    ALWAYS_REPROBE = 1 << 9,  // This is an encoder of last resort and we want to aggressively probe for a better one
   };
 
   struct encoder_platform_formats_t {
@@ -378,6 +379,10 @@ namespace video {
       std::vector<option_t> sdr_options;
       std::vector<option_t> hdr_options;
       std::vector<option_t> fallback_options;
+
+      // QP option to set in the case that CBR/VBR is not supported
+      // by the encoder. If CBR/VBR is guaranteed to be supported,
+      // don't specify this option to avoid wasteful encoder probing.
       std::optional<option_t> qp;
 
       std::string name;
@@ -594,7 +599,7 @@ namespace video {
       {},
       // Fallback options
       {},
-      std::nullopt,  // QP
+      std::nullopt,  // QP rate control fallback
       "av1_nvenc"s,
     },
     {
@@ -606,7 +611,7 @@ namespace video {
       {},
       // Fallback options
       {},
-      std::nullopt,  // QP
+      std::nullopt,  // QP rate control fallback
       "hevc_nvenc"s,
     },
     {
@@ -618,7 +623,7 @@ namespace video {
       {},
       // Fallback options
       {},
-      std::nullopt,  // QP
+      std::nullopt,  // QP rate control fallback
       "h264_nvenc"s,
     },
     PARALLEL_ENCODING | REF_FRAMES_INVALIDATION  // flags
@@ -659,7 +664,7 @@ namespace video {
       {},
       // Fallback options
       {},
-      std::nullopt,
+      std::nullopt,  // QP rate control fallback
       "av1_nvenc"s,
     },
     {
@@ -683,7 +688,7 @@ namespace video {
         { "profile"s, (int) nv::profile_hevc_e::main_10 },
       },
       {},  // Fallback options
-      std::nullopt,
+      std::nullopt,  // QP rate control fallback
       "hevc_nvenc"s,
     },
     {
@@ -704,7 +709,7 @@ namespace video {
       },
       {},  // HDR-specific options
       {},  // Fallback options
-      std::make_optional<encoder_t::option_t>({ "qp"s, &config::video.qp }),
+      std::nullopt,  // QP rate control fallback
       "h264_nvenc"s,
     },
     PARALLEL_ENCODING
@@ -734,7 +739,7 @@ namespace video {
       {},
       // Fallback options
       {},
-      std::make_optional<encoder_t::option_t>({ "qp"s, &config::video.qp }),
+      std::nullopt,  // QP rate control fallback
       "av1_qsv"s,
     },
     {
@@ -758,7 +763,7 @@ namespace video {
       },
       // Fallback options
       {},
-      std::make_optional<encoder_t::option_t>({ "qp"s, &config::video.qp }),
+      std::nullopt,  // QP rate control fallback
       "hevc_qsv"s,
     },
     {
@@ -785,7 +790,7 @@ namespace video {
       {
         { "low_power"s, 0 },  // Some old/low-end Intel GPUs don't support low power encoding
       },
-      std::make_optional<encoder_t::option_t>({ "qp"s, &config::video.qp }),
+      std::nullopt,  // QP rate control fallback
       "h264_qsv"s,
     },
     PARALLEL_ENCODING | CBR_WITH_VBR | RELAXED_COMPLIANCE | NO_RC_BUF_LIMIT
@@ -811,7 +816,7 @@ namespace video {
       {},  // SDR-specific options
       {},  // HDR-specific options
       {},  // Fallback options
-      std::make_optional<encoder_t::option_t>({ "qp_p"s, &config::video.qp }),
+      std::nullopt,  // QP rate control fallback
       "av1_amf"s,
     },
     {
@@ -832,7 +837,7 @@ namespace video {
       {},  // SDR-specific options
       {},  // HDR-specific options
       {},  // Fallback options
-      std::make_optional<encoder_t::option_t>({ "qp_p"s, &config::video.qp }),
+      std::nullopt,  // QP rate control fallback
       "hevc_amf"s,
     },
     {
@@ -856,7 +861,7 @@ namespace video {
       {
         { "usage"s, 2 /* AMF_VIDEO_ENCODER_USAGE_LOW_LATENCY */ },  // Workaround for https://github.com/GPUOpen-LibrariesAndSDKs/AMF/issues/410
       },
-      std::make_optional<encoder_t::option_t>({ "qp_p"s, &config::video.qp }),
+      std::nullopt,  // QP rate control fallback
       "h264_amf"s,
     },
     PARALLEL_ENCODING
@@ -882,7 +887,9 @@ namespace video {
       {},  // SDR-specific options
       {},  // HDR-specific options
       {},  // Fallback options
-      std::make_optional<encoder_t::option_t>("qp"s, &config::video.qp),
+
+      // QP rate control fallback
+      std::nullopt,
 
 #ifdef ENABLE_BROKEN_AV1_ENCODER
       // Due to bugs preventing on-demand IDR frames from working and very poor
@@ -907,7 +914,7 @@ namespace video {
       {},  // SDR-specific options
       {},  // HDR-specific options
       {},  // Fallback options
-      std::make_optional<encoder_t::option_t>("qp"s, &config::video.qp),
+      std::nullopt,  // QP rate control fallback
       "libx265"s,
     },
     {
@@ -919,10 +926,10 @@ namespace video {
       {},  // SDR-specific options
       {},  // HDR-specific options
       {},  // Fallback options
-      std::make_optional<encoder_t::option_t>("qp"s, &config::video.qp),
+      std::nullopt,  // QP rate control fallback
       "libx264"s,
     },
-    H264_ONLY | PARALLEL_ENCODING
+    H264_ONLY | PARALLEL_ENCODING | ALWAYS_REPROBE
   };
 
 #ifdef __linux__
@@ -2346,12 +2353,7 @@ namespace video {
   };
 
   int
-  validate_config(std::shared_ptr<platf::display_t> &disp, const encoder_t &encoder, const config_t &config) {
-    reset_display(disp, encoder.platform_formats->dev_type, config::video.output_name, config);
-    if (!disp) {
-      return -1;
-    }
-
+  validate_config(std::shared_ptr<platf::display_t> disp, const encoder_t &encoder, const config_t &config) {
     auto encode_device = make_encode_device(*disp, encoder, config);
     if (!encode_device) {
       return -1;
@@ -2472,7 +2474,13 @@ namespace video {
       if (disp->is_codec_supported(encoder.hevc.name, config_autoselect)) {
       retry_hevc:
         auto max_ref_frames_hevc = validate_config(disp, encoder, config_max_ref_frames);
-        auto autoselect_hevc = max_ref_frames_hevc >= 0 ? max_ref_frames_hevc : validate_config(disp, encoder, config_autoselect);
+
+        // If H.264 succeeded with max ref frames specified, assume that we can count on
+        // HEVC to also succeed with max ref frames specified if HEVC is supported.
+        auto autoselect_hevc = (max_ref_frames_hevc >= 0 || max_ref_frames_h264 >= 0) ?
+                                 max_ref_frames_hevc :
+                                 validate_config(disp, encoder, config_autoselect);
+
         if (autoselect_hevc < 0 && encoder.hevc.qp && encoder.hevc[encoder_t::CBR]) {
           // It's possible the encoder isn't accepting Constant Bit Rate. Turn off CBR and make another attempt
           encoder.hevc.capabilities.set();
@@ -2504,7 +2512,13 @@ namespace video {
       if (disp->is_codec_supported(encoder.av1.name, config_autoselect)) {
       retry_av1:
         auto max_ref_frames_av1 = validate_config(disp, encoder, config_max_ref_frames);
-        auto autoselect_av1 = max_ref_frames_av1 >= 0 ? max_ref_frames_av1 : validate_config(disp, encoder, config_autoselect);
+
+        // If H.264 succeeded with max ref frames specified, assume that we can count on
+        // AV1 to also succeed with max ref frames specified if AV1 is supported.
+        auto autoselect_av1 = (max_ref_frames_av1 >= 0 || max_ref_frames_h264 >= 0) ?
+                                max_ref_frames_av1 :
+                                validate_config(disp, encoder, config_autoselect);
+
         if (autoselect_av1 < 0 && encoder.av1.qp && encoder.av1[encoder_t::CBR]) {
           // It's possible the encoder isn't accepting Constant Bit Rate. Turn off CBR and make another attempt
           encoder.av1.capabilities.set();
@@ -2542,6 +2556,12 @@ namespace video {
       hevc.videoFormat = 1;
       av1.videoFormat = 2;
 
+      // Reset the display since we're switching from SDR to HDR
+      reset_display(disp, encoder.platform_formats->dev_type, config::video.output_name, config);
+      if (!disp) {
+        return false;
+      }
+
       // HDR is not supported with H.264. Don't bother even trying it.
       encoder.h264[flag] = flag != encoder_t::DYNAMIC_RANGE && validate_config(disp, encoder, h264) >= 0;
 
@@ -2579,6 +2599,11 @@ namespace video {
   probe_encoders() {
     auto encoder_list = encoders;
 
+    // If we already have a good encoder, check to see if another probe is required
+    if (chosen_encoder && !(chosen_encoder->flags & ALWAYS_REPROBE) && !platf::needs_encoder_reenumeration()) {
+      return 0;
+    }
+
     // Restart encoder selection
     auto previous_encoder = chosen_encoder;
     chosen_encoder = nullptr;