From ad1ef92623a1188beda3e5ffb0ed083003873f75 Mon Sep 17 00:00:00 2001
From: Cameron Gutman <aicommander@gmail.com>
Date: Mon, 8 Jul 2024 10:05:35 -0500
Subject: [PATCH] Fix rate control for AMD cards using VAAPI (#2821)

---
 src/platform/common.h        |  9 +++++++++
 src/platform/linux/vaapi.cpp | 10 ++++++++++
 src/video.cpp                |  4 ++++
 3 files changed, 23 insertions(+)

diff --git a/src/platform/common.h b/src/platform/common.h
index 727586c6b01..f42d366dac0 100644
--- a/src/platform/common.h
+++ b/src/platform/common.h
@@ -27,6 +27,8 @@ struct sockaddr;
 struct AVFrame;
 struct AVBufferRef;
 struct AVHWFramesContext;
+struct AVCodecContext;
+struct AVDictionary;
 
 // Forward declarations of boost classes to avoid having to include boost headers
 // here, which results in issues with Windows.h and WinSock2.h include order.
@@ -398,6 +400,13 @@ namespace platf {
     virtual void
     init_hwframes(AVHWFramesContext *frames) {};
 
+    /**
+     * @brief Provides a hook for allow platform-specific code to adjust codec options.
+     * @note Implementations may set or modify codec options prior to codec initialization.
+     */
+    virtual void
+    init_codec_options(AVCodecContext *ctx, AVDictionary *options) {};
+
     /**
      * @brief Prepare to derive a context.
      * @note Implementations may make modifications required before context derivation
diff --git a/src/platform/linux/vaapi.cpp b/src/platform/linux/vaapi.cpp
index cd1d81f61d4..ada6370034a 100644
--- a/src/platform/linux/vaapi.cpp
+++ b/src/platform/linux/vaapi.cpp
@@ -129,6 +129,16 @@ namespace va {
       return 0;
     }
 
+    void
+    init_codec_options(AVCodecContext *ctx, AVDictionary *options) override {
+      // Don't set the RC buffer size when using H.264 on Intel GPUs. It causes
+      // major encoding quality degradation.
+      auto vendor = vaQueryVendorString(va_display);
+      if (ctx->codec_id != AV_CODEC_ID_H264 || (vendor && !strstr(vendor, "Intel"))) {
+        ctx->rc_buffer_size = ctx->bit_rate * ctx->framerate.den / ctx->framerate.num;
+      }
+    }
+
     int
     set_frame(AVFrame *frame, AVBufferRef *hw_frames_ctx_buf) override {
       this->hwframe.reset(frame);
diff --git a/src/video.cpp b/src/video.cpp
index 10c91d1ef38..7758d68a9b5 100644
--- a/src/video.cpp
+++ b/src/video.cpp
@@ -859,6 +859,7 @@ namespace video {
       std::make_optional<encoder_t::option_t>("qp"s, &config::video.qp),
       "h264_vaapi"s,
     },
+    // RC buffer size will be set in platform code if supported
     LIMITED_GOP_SIZE | PARALLEL_ENCODING | SINGLE_SLICE_ONLY | NO_RC_BUF_LIMIT
   };
 #endif
@@ -1610,6 +1611,9 @@ namespace video {
         return nullptr;
       }
 
+      // Allow the encoding device a final opportunity to set/unset or override any options
+      encode_device->init_codec_options(ctx.get(), options);
+
       if (auto status = avcodec_open2(ctx.get(), codec, &options)) {
         char err_str[AV_ERROR_MAX_STRING_SIZE] { 0 };