fix: correct vulkan settings

janhq · Feb 20, 2024 · dcdebbd · dcdebbd
1 parent b52ce26
commit dcdebbd
Show file tree

Hide file tree

Showing 5 changed files with 29 additions and 12 deletions.
diff --git a/extensions/inference-nitro-extension/src/node/execute.ts b/extensions/inference-nitro-extension/src/node/execute.ts
@@ -5,6 +5,7 @@ import { GPU_INFO_FILE } from './accelerator'
 export interface NitroExecutableOptions {
   executablePath: string
   cudaVisibleDevices: string
+  vkVisibleDevices: string
 }
 /**
  * Find which executable file to run based on the current platform.
@@ -13,6 +14,7 @@ export interface NitroExecutableOptions {
 export const executableNitroFile = (): NitroExecutableOptions => {
   let binaryFolder = path.join(__dirname, '..', 'bin') // Current directory by default
   let cudaVisibleDevices = ''
+  let vkVisibleDevices = ''
   let binaryName = 'nitro'
   /**
    * The binary folder is different for each platform.
@@ -25,17 +27,17 @@ export const executableNitroFile = (): NitroExecutableOptions => {
     if (gpuInfo['run_mode'] === 'cpu') {
       binaryFolder = path.join(binaryFolder, 'win-cpu')
     } else {
-      if (gpuInfo['cuda'].version === '11') {
+      if (gpuInfo['cuda']?.version === '11') {
         binaryFolder = path.join(binaryFolder, 'win-cuda-11-7')
       } else {
         binaryFolder = path.join(binaryFolder, 'win-cuda-12-0')
       }
       cudaVisibleDevices = gpuInfo['gpus_in_use'].join(',')
     }
-    if (gpuInfo['vulkan'].enabled) {
+    if (gpuInfo['vulkan'] === true) {
       binaryFolder = path.join(__dirname, '..', 'bin')
       binaryFolder = path.join(binaryFolder, 'win-vulkan')
-      process.env.VULKAN_DEVICE_INDEX = gpuInfo['vulkan'].gpu_in_use.toString()
+      vkVisibleDevices = gpuInfo['gpus_in_use'].toString()
     }
     binaryName = 'nitro.exe'
   } else if (process.platform === 'darwin') {
@@ -55,21 +57,23 @@ export const executableNitroFile = (): NitroExecutableOptions => {
     if (gpuInfo['run_mode'] === 'cpu') {
       binaryFolder = path.join(binaryFolder, 'linux-cpu')
     } else {
-      if (gpuInfo['cuda'].version === '11') {
+      if (gpuInfo['cuda']?.version === '11') {
         binaryFolder = path.join(binaryFolder, 'linux-cuda-11-7')
       } else {
         binaryFolder = path.join(binaryFolder, 'linux-cuda-12-0')
       }
       cudaVisibleDevices = gpuInfo['gpus_in_use'].join(',')
     }
-    if (gpuInfo['vulkan'].enabled) {
+
+    if (gpuInfo['vulkan'] === true) {
       binaryFolder = path.join(__dirname, '..', 'bin')
       binaryFolder = path.join(binaryFolder, 'win-vulkan')
-      process.env.VULKAN_DEVICE_INDEX = gpuInfo['vulkan'].gpu_in_use.toString()
+      vkVisibleDevices = gpuInfo['gpus_in_use'].toString()
     }
   }
   return {
     executablePath: path.join(binaryFolder, binaryName),
     cudaVisibleDevices,
+    vkVisibleDevices,
   }
 }
diff --git a/extensions/inference-nitro-extension/src/node/index.ts b/extensions/inference-nitro-extension/src/node/index.ts
@@ -345,6 +345,10 @@ function spawnNitroProcess(): Promise<any> {
         env: {
           ...process.env,
           CUDA_VISIBLE_DEVICES: executableOptions.cudaVisibleDevices,
+          // Vulkan - Support 1 device at a time for now
+          ...(executableOptions.vkVisibleDevices?.length > 0 && {
+            GGML_VULKAN_DEVICE: executableOptions.vkVisibleDevices[0],
+          }),
         },
       }
     )

diff --git a/extensions/monitoring-extension/src/module.ts b/extensions/monitoring-extension/src/module.ts
@@ -37,7 +37,7 @@ const getCurrentLoad = () =>
       }
       if (data.run_mode === 'gpu' && data.gpus_in_use.length > 0) {
         const gpuIds = data['gpus_in_use'].join(',')
-        if (gpuIds !== '') {
+        if (gpuIds !== '' && data['vulkan'] !== true) {
           exec(
             `nvidia-smi --query-gpu=index,name,temperature.gpu,utilization.gpu,memory.total,memory.free,utilization.memory --format=csv,noheader,nounits --id=${gpuIds}`,
             (error, stdout, stderr) => {

diff --git a/web/hooks/useSettings.ts b/web/hooks/useSettings.ts
@@ -48,16 +48,19 @@ export const useSettings = () => {
     runMode,
     notify,
     gpusInUse,
+    vulkan,
   }: {
     runMode?: string | undefined
     notify?: boolean | undefined
     gpusInUse?: string[] | undefined
+    vulkan?: boolean | undefined
   }) => {
     const settingsFile = await joinPath(['file://settings', 'settings.json'])
     const settings = await readSettings()
     if (runMode != null) settings.run_mode = runMode
     if (notify != null) settings.notify = notify
     if (gpusInUse != null) settings.gpus_in_use = gpusInUse
+    if (vulkan != null) settings.vulkan = vulkan
     await fs.writeFileSync(settingsFile, JSON.stringify(settings))
   }
 

diff --git a/web/screens/Settings/Advanced/index.tsx b/web/screens/Settings/Advanced/index.tsx
@@ -93,6 +93,7 @@ const Advanced = () => {
       const settings = await readSettings()
       setGpuEnabled(settings.run_mode === 'gpu' && settings.gpus?.length > 0)
       setGpusInUse(settings.gpus_in_use || [])
+      setVulkanEnabled(settings.vulkan || false)
       if (settings.gpus) {
         setGpuList(settings.gpus)
       }
@@ -342,13 +343,18 @@ const Advanced = () => {
             </div>
             <p className="text-xs leading-relaxed">
               Enable Vulkan with AMD GPU/APU and Intel Arc GPU for better model
-              performance. 0 - iGPU, n positive - eGPU
+              performance.
             </p>
           </div>
-          {vulkanEnabled && (
-            <Input value={vulkanEnabled ? 1 : 0} className="w-[60px] pr-8" />
-          )}
-          <Switch checked={vulkanEnabled} onCheckedChange={setVulkanEnabled} />
+
+          <Switch
+            checked={vulkanEnabled}
+            onCheckedChange={(e) => {
+              stopModel()
+              saveSettings({ vulkan: e, gpusInUse: [] })
+              setVulkanEnabled(e)
+            }}
+          />
         </div>
       )}