fix: black screen and crashes on nvidia GPUs (#695)

Code/client/NvidiaUtil.cpp

@@ -0,0 +1,16 @@
+#include "NvidiaUtil.h"
+#include <d3d11.h>
+
+bool IsNvidiaOverlayLoaded()
+{
+    return GetModuleHandleW(L"nvspcap64.dll");
+}
+
+// This makes the Nvidia overlay happy.
+// The call to D3D11CreateDevice probably causes some of their
+// internal hooks to be called and do the required init work before the game window opens.
+HRESULT CreateEarlyDxDevice(ID3D11Device* apOutDevice, D3D_FEATURE_LEVEL* apOutFeatureLevel)
+{
+    return D3D11CreateDevice(nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, 0, nullptr, 0, D3D11_SDK_VERSION, &apOutDevice,
+                             apOutFeatureLevel, nullptr);
+}

Code/client/NvidiaUtil.h

@@ -0,0 +1,8 @@
+#pragma once
+
+#include <Windows.h>
+#include <d3d11.h>
+
+bool IsNvidiaOverlayLoaded();
+
+HRESULT CreateEarlyDxDevice(ID3D11Device* apOutDevice, D3D_FEATURE_LEVEL* apOutFeatureLevel);

Code/client/TiltedOnlineApp.cpp

@@ -7,6 +7,7 @@
 
 #include <World.h>
 
+#include <spdlog/spdlog.h>
 #include <spdlog/sinks/rotating_file_sink.h>
 #include <spdlog/sinks/stdout_color_sinks.h>
 
@@ -17,6 +18,7 @@
 #include <Services/DiscordService.h>
 
 #include <ScriptExtender.h>
+#include <NvidiaUtil.h>
 
 using TiltedPhoques::Debug;
 
@@ -56,12 +58,18 @@ bool TiltedOnlineApp::BeginMain()
     World::Get().ctx().emplace<RenderSystemD3D11>(World::Get().ctx().at<OverlayService>(), World::Get().ctx().at<ImguiService>());
 
     LoadScriptExender();
+
+    if (IsNvidiaOverlayLoaded())
+        ApplyNvidiaFix();
+
     return true;
 }
 
 bool TiltedOnlineApp::EndMain()
 {
     UninstallHooks();
+    if (m_pDevice)
+        m_pDevice->Release();
 
     return true;
 }
@@ -107,3 +115,14 @@ void TiltedOnlineApp::InstallHooks2()
 void TiltedOnlineApp::UninstallHooks()
 {
 }
+
+void TiltedOnlineApp::ApplyNvidiaFix() noexcept
+{
+    auto d3dFeatureLevel = D3D_FEATURE_LEVEL_11_0;
+    HRESULT hr = CreateEarlyDxDevice(m_pDevice, &d3dFeatureLevel);
+    if (FAILED(hr))
+        spdlog::error("D3D11CreateDevice failed. Detected an NVIDIA GPU, error code={0:x}", hr);
+
+    if (d3dFeatureLevel < D3D_FEATURE_LEVEL_11_0)
+        spdlog::warn("Unexpected D3D11 feature level detected (< 11.0), may cause issues");
+}

Code/client/TiltedOnlineApp.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <BranchInfo.h>
+#include <d3d11.h>
 
 #if (!IS_MASTER)
 #include "CrashHandler.h"
@@ -35,9 +36,11 @@ struct TiltedOnlineApp final : App
     void UninstallHooks();
 
 private:
+    void ApplyNvidiaFix() noexcept;
 #if (!IS_MASTER)
     CrashHandler m_crashHandler;
 #else
     //ScopedCrashHandler m_crashHandler;
 #endif
+    ID3D11Device* m_pDevice = nullptr;
 };

Code/immersive_launcher/Launcher.cpp

@@ -7,7 +7,6 @@
 #include "loader/PathRerouting.h"
 
 #include "Utils/Error.h"
-#include "Utils/NvidiaUtil.h"
 #include "Utils/FileVersion.inl"
 
 #include "oobe/PathSelection.h"
@@ -17,7 +16,6 @@
 #include "base/dialogues/win/TaskDialog.h"
 
 #include <BranchInfo.h>
-#include <spdlog/spdlog.h>
 
 // These symbols are defined within the client code skyrimtogetherclient
 extern void InstallStartHook();
@@ -103,13 +101,6 @@ int StartUp(int argc, char** argv)
     if (!oobe::SelectInstall(askSelect))
         DIE_NOW(L"Failed to select game install.");
 
-    if (IsNvidiaOverlayLoaded())
-    {
-        HRESULT hr = CreateEarlyDxSwapChain();
-        if (FAILED(hr))
-            spdlog::error("D3D11CreateDeviceAndSwapChain failed! (Detected an NVIDIA GPU, error code={})", hr);
-    }
-
     // Bind path environment.
     loader::InstallPathRouting(LC->gamePath);
     steam::Load(LC->gamePath);

Code/immersive_launcher/Main.cpp

@@ -22,8 +22,8 @@ extern "C"
 auto kSystemPreloadDlls = {
     L"\\dinput8.dll",   // < Skyrim early init hook
     L"\\dsound.dll",    // < breaks DSound init in game code
+    L"\\nvspcap64.dll", // < Nvidia overlay, needs to be loaded before d3d11 device creation
                         // < X360CE v3 is buggy with COM hooks
-    L"\\nvspcap64.dll", // < Nvidia overlay, needs to be loaded before the swap chain creation
     L"\\xinput9_1_0.dll", L"\\xinput1_1.dll", L"\\xinput1_2.dll", L"\\xinput1_3.dll", L"\\xinput1_4.dll", L"\\version.dll"};
 
 static void PreloadSystemDlls()