build(linux) make vaapi optional without dlopen (#1979)

Co-authored-by: ReenigneArcher <[email protected]>

cmake/FindLibva.cmake

@@ -0,0 +1,70 @@
+# - Try to find Libva
+# This module defines the following variables:
+#
+# * LIBVA_FOUND - The component was found
+# * LIBVA_INCLUDE_DIRS - The component include directory
+# * LIBVA_LIBRARIES - The component library Libva
+# * LIBVA_DRM_LIBRARIES - The component library Libva DRM
+
+# Use pkg-config to get the directories and then use these values in the
+# find_path() and find_library() calls
+# cmake-format: on
+
+find_package(PkgConfig QUIET)
+if(PKG_CONFIG_FOUND)
+    pkg_check_modules(_LIBVA libva)
+    pkg_check_modules(_LIBVA_DRM libva-drm)
+endif()
+
+find_path(
+        LIBVA_INCLUDE_DIR
+        NAMES va/va.h va/va_drm.h
+        HINTS ${_LIBVA_INCLUDE_DIRS}
+        PATHS /usr/include /usr/local/include /opt/local/include)
+
+find_library(
+        LIBVA_LIB
+        NAMES ${_LIBVA_LIBRARIES} libva
+        HINTS ${_LIBVA_LIBRARY_DIRS}
+        PATHS /usr/lib /usr/local/lib /opt/local/lib)
+
+find_library(
+        LIBVA_DRM_LIB
+        NAMES ${_LIBVA_DRM_LIBRARIES} libva-drm
+        HINTS ${_LIBVA_DRM_LIBRARY_DIRS}
+        PATHS /usr/lib /usr/local/lib /opt/local/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Libva REQUIRED_VARS LIBVA_INCLUDE_DIR LIBVA_LIB LIBVA_DRM_LIB)
+mark_as_advanced(LIBVA_INCLUDE_DIR LIBVA_LIB LIBVA_DRM_LIB)
+
+if(LIBVA_FOUND)
+    set(LIBVA_INCLUDE_DIRS ${LIBVA_INCLUDE_DIR})
+    set(LIBVA_LIBRARIES ${LIBVA_LIB})
+    set(LIBVA_DRM_LIBRARIES ${LIBVA_DRM_LIB})
+
+    if(NOT TARGET Libva::va)
+        if(IS_ABSOLUTE "${LIBVA_LIBRARIES}")
+            add_library(Libva::va UNKNOWN IMPORTED)
+            set_target_properties(Libva::va PROPERTIES IMPORTED_LOCATION "${LIBVA_LIBRARIES}")
+        else()
+            add_library(Libva::va INTERFACE IMPORTED)
+            set_target_properties(Libva::va PROPERTIES IMPORTED_LIBNAME "${LIBVA_LIBRARIES}")
+        endif()
+
+        set_target_properties(Libva::va PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${LIBVA_INCLUDE_DIRS}")
+    endif()
+
+    if(NOT TARGET Libva::drm)
+        if(IS_ABSOLUTE "${LIBVA_DRM_LIBRARIES}")
+            add_library(Libva::drm UNKNOWN IMPORTED)
+            set_target_properties(Libva::drm PROPERTIES IMPORTED_LOCATION "${LIBVA_DRM_LIBRARIES}")
+        else()
+            add_library(Libva::drm INTERFACE IMPORTED)
+            set_target_properties(Libva::drm PROPERTIES IMPORTED_LIBNAME "${LIBVA_DRM_LIBRARIES}")
+        endif()
+
+        set_target_properties(Libva::drm PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${LIBVA_INCLUDE_DIRS}")
+    endif()
+
+endif()

cmake/compile_definitions/linux.cmake

@@ -120,6 +120,21 @@ elseif(NOT LIBDRM_FOUND)
     message(WARNING "Missing libcap")
 endif()
 
+# vaapi
+if(${SUNSHINE_ENABLE_VAAPI})
+    find_package(Libva)
+else()
+    set(LIBVA_FOUND OFF)
+endif()
+if(LIBVA_FOUND)
+    add_compile_definitions(SUNSHINE_BUILD_VAAPI)
+    include_directories(SYSTEM ${LIBVA_INCLUDE_DIR})
+    list(APPEND PLATFORM_LIBRARIES ${LIBVA_LIBRARIES} ${LIBVA_DRM_LIBRARIES})
+    list(APPEND PLATFORM_TARGET_FILES
+            src/platform/linux/vaapi.h
+            src/platform/linux/vaapi.cpp)
+endif()
+
 # wayland
 if(${SUNSHINE_ENABLE_WAYLAND})
     find_package(Wayland)
@@ -167,8 +182,12 @@ if(X11_FOUND)
             src/platform/linux/x11grab.cpp)
 endif()
 
-if(NOT ${CUDA_FOUND} AND NOT ${WAYLAND_FOUND} AND NOT ${X11_FOUND} AND NOT (${LIBDRM_FOUND} AND ${LIBCAP_FOUND}))
-    message(FATAL_ERROR "Couldn't find either x11, wayland, cuda or (libdrm and libcap)")
+if(NOT ${CUDA_FOUND}
+        AND NOT ${WAYLAND_FOUND}
+        AND NOT ${X11_FOUND}
+        AND NOT (${LIBDRM_FOUND} AND ${LIBCAP_FOUND})
+        AND NOT ${LIBVA_FOUND})
+    message(FATAL_ERROR "Couldn't find either cuda, wayland, x11, (libdrm and libcap), or libva")
 endif()
 
 # tray icon
@@ -206,8 +225,6 @@ endif()
 
 list(APPEND PLATFORM_TARGET_FILES
         src/platform/linux/publish.cpp
-        src/platform/linux/vaapi.h
-        src/platform/linux/vaapi.cpp
         src/platform/linux/graphics.h
         src/platform/linux/graphics.cpp
         src/platform/linux/misc.h

cmake/prep/options.cmake

@@ -26,6 +26,8 @@ elseif(UNIX)  # Linux
             "Enable cuda specific code." ON)
     option(SUNSHINE_ENABLE_DRM
             "Enable KMS grab if available." ON)
+    option(SUNSHINE_ENABLE_VAAPI
+            "Enable building vaapi specific code." ON)
     option(SUNSHINE_ENABLE_WAYLAND
             "Enable building wayland specific code." ON)
     option(SUNSHINE_ENABLE_X11

docs/source/building/linux.rst

@@ -32,7 +32,7 @@ Install Requirements
           libopus-dev \
           libpulse-dev \
           libssl-dev \
-          libva-dev \
+          libva-dev \  # VA-API
           libvdpau-dev \
           libwayland-dev \  # Wayland
           libx11-dev \  # X11
@@ -67,7 +67,7 @@ Install Requirements
           libdrm-devel \
           libevdev-devel \
           libnotify-devel \
-          libva-devel \
+          libva-devel \  # VA-API
           libvdpau-devel \
           libX11-devel \  # X11
           libxcb-devel \  # X11
@@ -115,7 +115,7 @@ Install Requirements
           libopus-dev \
           libpulse-dev \
           libssl-dev \
-          libva-dev \
+          libva-dev \  # VA-API
           libvdpau-dev \
           libwayland-dev \  # Wayland
           libx11-dev \  # X11
@@ -165,6 +165,7 @@ Install Requirements
           libopus-dev \
           libpulse-dev \
           libssl-dev \
+          libva-dev \  # VA-API
           libwayland-dev \  # Wayland
           libx11-dev \  # X11
           libxcb-shm0-dev \  # X11

src/platform/linux/kmsgrab.cpp

@@ -793,9 +793,11 @@ namespace platf {
 
       std::unique_ptr<avcodec_encode_device_t>
       make_avcodec_encode_device(pix_fmt_e pix_fmt) override {
+#ifdef SUNSHINE_BUILD_VAAPI
         if (mem_type == mem_type_e::vaapi) {
           return va::make_avcodec_encode_device(width, height, false);
         }
+#endif
 
         return std::make_unique<avcodec_encode_device_t>();
       }
@@ -862,9 +864,11 @@ namespace platf {
 
       std::unique_ptr<avcodec_encode_device_t>
       make_avcodec_encode_device(pix_fmt_e pix_fmt) override {
+#ifdef SUNSHINE_BUILD_VAAPI
         if (mem_type == mem_type_e::vaapi) {
           return va::make_avcodec_encode_device(width, height, dup(card.render_fd.el), img_offset_x, img_offset_y, true);
         }
+#endif
 
         BOOST_LOG(error) << "Unsupported pixel format for egl::display_vram_t: "sv << platf::from_pix_fmt(pix_fmt);
         return nullptr;
@@ -977,7 +981,11 @@ namespace platf {
           return -1;
         }
 
+#ifdef SUNSHINE_BUILD_VAAPI
         if (!va::validate(card.render_fd.el)) {
+#else
+        if (true) {
+#endif
           BOOST_LOG(warning) << "Monitor "sv << display_name << " doesn't support hardware encoding. Reverting back to GPU -> RAM -> GPU"sv;
           return -1;
         }

src/platform/linux/misc.cpp

@@ -724,7 +724,6 @@ namespace platf {
   init() {
     // These are allowed to fail.
     gbm::init();
-    va::init();
 
     window_system = window_system_e::NONE;
 #ifdef SUNSHINE_BUILD_WAYLAND