feat: add vulkan support on windows and linux

extensions/inference-nitro-extension/bin/version.txt

@@ -1 +1 @@
-0.3.12
+0.3.12

extensions/inference-nitro-extension/download.bat

@@ -1,3 +1,3 @@
 @echo off
 set /p NITRO_VERSION=<./bin/version.txt
-.\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/win-cuda-12-0 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/win-cuda-11-7 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64.tar.gz -e --strip 1 -o ./bin/win-cpu
+.\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/win-cuda-12-0 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/win-cuda-11-7 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64.tar.gz -e --strip 1 -o ./bin/win-cpu && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-vulkan.tar.gz -e --strip 1 -o ./bin/win-vulkan

extensions/inference-nitro-extension/package.json

@@ -8,7 +8,7 @@
   "license": "AGPL-3.0",
   "scripts": {
     "build": "tsc --module commonjs && rollup -c rollup.config.ts",
-    "downloadnitro:linux": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64.tar.gz -e --strip 1 -o ./bin/linux-cpu && chmod +x ./bin/linux-cpu/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/linux-cuda-12-0 && chmod +x ./bin/linux-cuda-12-0/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/linux-cuda-11-7 && chmod +x ./bin/linux-cuda-11-7/nitro",
+    "downloadnitro:linux": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64.tar.gz -e --strip 1 -o ./bin/linux-cpu && chmod +x ./bin/linux-cpu/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/linux-cuda-12-0 && chmod +x ./bin/linux-cuda-12-0/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/linux-cuda-11-7 && chmod +x ./bin/linux-cuda-11-7/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-vulkan-11-7.tar.gz -e --strip 1 -o ./bin/linux-vulkan && chmod +x ./bin/linux-vulkan/nitro ",
     "downloadnitro:darwin": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-arm64.tar.gz -e --strip 1 -o ./bin/mac-arm64 && chmod +x ./bin/mac-arm64/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-amd64.tar.gz -e --strip 1 -o ./bin/mac-x64 && chmod +x ./bin/mac-x64/nitro",
     "downloadnitro:win32": "download.bat",
     "downloadnitro": "run-script-os",

extensions/inference-nitro-extension/src/node/nvidia.ts → extensions/inference-nitro-extension/src/node/accelerator.ts

@@ -5,6 +5,7 @@ import { getJanDataFolderPath } from '@janhq/core/node'
 
 /**
  * Default GPU settings
+ * TODO: This needs to be refactored to support multiple accelerators
  **/
 const DEFALT_SETTINGS = {
   notify: true,
@@ -21,12 +22,17 @@ const DEFALT_SETTINGS = {
   gpu_highest_vram: '',
   gpus_in_use: [],
   is_initial: true,
+  // TODO: This needs to be set based on user toggle in settings
+  vulkan: {
+    enabled: true,
+    gpu_in_use: '1',
+  },
 }
 
 /**
  * Path to the settings file
  **/
-export const NVIDIA_INFO_FILE = path.join(
+export const GPU_INFO_FILE = path.join(
   getJanDataFolderPath(),
   'settings',
   'settings.json'
@@ -52,10 +58,10 @@ export async function updateNvidiaInfo() {
   if (process.platform !== 'darwin') {
     let data
     try {
-      data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8'))
+      data = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8'))
     } catch (error) {
       data = DEFALT_SETTINGS
-      writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2))
+      writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2))
     }
     updateNvidiaDriverInfo()
     updateGpuInfo()
@@ -79,7 +85,7 @@ export async function updateNvidiaDriverInfo(): Promise<void> {
   exec(
     'nvidia-smi --query-gpu=driver_version --format=csv,noheader',
     (error, stdout) => {
-      let data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8'))
+      let data = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8'))
 
       if (!error) {
         const firstLine = stdout.split('\n')[0].trim()
@@ -89,7 +95,7 @@ export async function updateNvidiaDriverInfo(): Promise<void> {
         data['nvidia_driver'].exist = false
       }
 
-      writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2))
+      writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2))
       Promise.resolve()
     }
   )
@@ -161,7 +167,7 @@ export async function updateGpuInfo(): Promise<void> {
   exec(
     'nvidia-smi --query-gpu=index,memory.total,name --format=csv,noheader,nounits',
     (error, stdout) => {
-      let data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8'))
+      let data = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8'))
 
       if (!error) {
         // Get GPU info and gpu has higher memory first
@@ -192,7 +198,7 @@ export async function updateGpuInfo(): Promise<void> {
       }
 
       data = updateCudaExistence(data)
-      writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2))
+      writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2))
       Promise.resolve()
     }
   )

extensions/inference-nitro-extension/src/node/execute.ts

@@ -1,6 +1,6 @@
 import { readFileSync } from 'fs'
 import * as path from 'path'
-import { NVIDIA_INFO_FILE } from './nvidia'
+import { GPU_INFO_FILE } from './accelerator'
 
 export interface NitroExecutableOptions {
   executablePath: string
@@ -19,18 +19,23 @@ export const executableNitroFile = (): NitroExecutableOptions => {
    */
   if (process.platform === 'win32') {
     /**
-     *  For Windows: win-cpu, win-cuda-11-7, win-cuda-12-0
+     *  For Windows: win-cpu, win-vulkan, win-cuda-11-7, win-cuda-12-0
      */
-    let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8'))
-    if (nvidiaInfo['run_mode'] === 'cpu') {
+    let gpuInfo = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8'))
+    if (gpuInfo['run_mode'] === 'cpu') {
       binaryFolder = path.join(binaryFolder, 'win-cpu')
     } else {
-      if (nvidiaInfo['cuda'].version === '11') {
+      if (gpuInfo['cuda'].version === '11') {
         binaryFolder = path.join(binaryFolder, 'win-cuda-11-7')
       } else {
         binaryFolder = path.join(binaryFolder, 'win-cuda-12-0')
       }
-      cudaVisibleDevices = nvidiaInfo['gpus_in_use'].join(',')
+      cudaVisibleDevices = gpuInfo['gpus_in_use'].join(',')
+    }
+    if (gpuInfo['vulkan'].enabled) {
+      binaryFolder = path.join(__dirname, '..', 'bin')
+      binaryFolder = path.join(binaryFolder, 'win-vulkan')
+      process.env.VULKAN_DEVICE_INDEX = gpuInfo['vulkan'].gpu_in_use.toString()
     }
     binaryName = 'nitro.exe'
   } else if (process.platform === 'darwin') {
@@ -44,18 +49,23 @@ export const executableNitroFile = (): NitroExecutableOptions => {
     }
   } else {
     /**
-     *  For Linux: linux-cpu, linux-cuda-11-7, linux-cuda-12-0
+     *  For Linux: linux-cpu, linux-vulkan, linux-cuda-11-7, linux-cuda-12-0
      */
-    let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8'))
-    if (nvidiaInfo['run_mode'] === 'cpu') {
+    let gpuInfo = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8'))
+    if (gpuInfo['run_mode'] === 'cpu') {
       binaryFolder = path.join(binaryFolder, 'linux-cpu')
     } else {
-      if (nvidiaInfo['cuda'].version === '11') {
+      if (gpuInfo['cuda'].version === '11') {
         binaryFolder = path.join(binaryFolder, 'linux-cuda-11-7')
       } else {
         binaryFolder = path.join(binaryFolder, 'linux-cuda-12-0')
       }
-      cudaVisibleDevices = nvidiaInfo['gpus_in_use'].join(',')
+      cudaVisibleDevices = gpuInfo['gpus_in_use'].join(',')
+    }
+    if (gpuInfo['vulkan'].enabled) {
+      binaryFolder = path.join(__dirname, '..', 'bin')
+      binaryFolder = path.join(binaryFolder, 'win-vulkan')
+      process.env.VULKAN_DEVICE_INDEX = gpuInfo['vulkan'].gpu_in_use.toString()
     }
   }
   return {

extensions/inference-nitro-extension/src/node/index.ts

@@ -4,7 +4,7 @@ import { ChildProcessWithoutNullStreams, spawn } from 'child_process'
 import tcpPortUsed from 'tcp-port-used'
 import fetchRT from 'fetch-retry'
 import { log, getSystemResourceInfo } from '@janhq/core/node'
-import { getNitroProcessInfo, updateNvidiaInfo } from './nvidia'
+import { getNitroProcessInfo, updateNvidiaInfo } from './accelerator'
 import {
   Model,
   InferenceEngine,

web/containers/ServerLogs/index.tsx

@@ -229,4 +229,4 @@ const ServerLogs = (props: ServerLogsProps) => {
   )
 }
 
-export default ServerLogs
+export default ServerLogs

web/context/FeatureToggle.tsx

@@ -5,7 +5,9 @@ interface FeatureToggleContextType {
   ignoreSSL: boolean
   proxy: string
   proxyEnabled: boolean
+  vulkanEnabled: boolean
   setExperimentalFeature: (on: boolean) => void
+  setVulkanEnabled: (on: boolean) => void
   setIgnoreSSL: (on: boolean) => void
   setProxy: (value: string) => void
   setProxyEnabled: (on: boolean) => void
@@ -16,7 +18,9 @@ const initialContext: FeatureToggleContextType = {
   ignoreSSL: false,
   proxy: '',
   proxyEnabled: false,
+  vulkanEnabled: false,
   setExperimentalFeature: () => {},
+  setVulkanEnabled: () => {},
   setIgnoreSSL: () => {},
   setProxy: () => {},
   setProxyEnabled: () => {},
@@ -31,13 +35,15 @@ export default function FeatureToggleWrapper({
   children: ReactNode
 }) {
   const EXPERIMENTAL_FEATURE = 'experimentalFeature'
+  const VULKAN_ENABLED = 'vulkanEnabled'
   const IGNORE_SSL = 'ignoreSSLFeature'
   const HTTPS_PROXY_FEATURE = 'httpsProxyFeature'
   const PROXY_FEATURE_ENABLED = 'proxyFeatureEnabled'
 
   const [experimentalFeature, directSetExperimentalFeature] =
     useState<boolean>(false)
   const [proxyEnabled, directSetProxyEnabled] = useState<boolean>(false)
+  const [vulkanEnabled, directEnableVulkan] = useState<boolean>(false)
   const [ignoreSSL, directSetIgnoreSSL] = useState<boolean>(false)
   const [proxy, directSetProxy] = useState<string>('')
 
@@ -57,6 +63,11 @@ export default function FeatureToggleWrapper({
     directSetExperimentalFeature(on)
   }
 
+  const setVulkanEnabled = (on: boolean) => {
+    localStorage.setItem(VULKAN_ENABLED, on ? 'true' : 'false')
+    directEnableVulkan(on)
+  }
+
   const setIgnoreSSL = (on: boolean) => {
     localStorage.setItem(IGNORE_SSL, on ? 'true' : 'false')
     directSetIgnoreSSL(on)
@@ -79,7 +90,9 @@ export default function FeatureToggleWrapper({
         ignoreSSL,
         proxy,
         proxyEnabled,
+        vulkanEnabled,
         setExperimentalFeature,
+        setVulkanEnabled,
         setIgnoreSSL,
         setProxy,
         setProxyEnabled,

web/screens/Settings/Advanced/index.tsx

@@ -58,6 +58,8 @@ const Advanced = () => {
     setProxy,
     proxyEnabled,
     setProxyEnabled,
+    vulkanEnabled,
+    setVulkanEnabled,
   } = useContext(FeatureToggleContext)
   const [partialProxy, setPartialProxy] = useState<string>(proxy)
   const [gpuEnabled, setGpuEnabled] = useState<boolean>(false)
@@ -169,12 +171,12 @@ const Advanced = () => {
             <div className="space-y-1.5">
               <div className="flex gap-x-2">
                 <h6 className="text-sm font-semibold capitalize">
-                  GPU Acceleration
+                  NVIDIA GPU Acceleration
                 </h6>
               </div>
               <p className="pr-8 leading-relaxed">
-                Enable to enhance model performance by utilizing your devices
-                GPU for acceleration. Read{' '}
+                Enable to enhance model performance by utilizing your NVIDIA GPU
+                devices for acceleration. Read{' '}
                 <span>
                   {' '}
                   <span
@@ -202,7 +204,7 @@ const Advanced = () => {
                   className="max-w-[240px]"
                 >
                   <span>
-                    Disabling GPU Acceleration may result in reduced
+                    Disabling NVIDIA GPU Acceleration may result in reduced
                     performance. It is recommended to keep this enabled for
                     optimal user experience.
                   </span>
@@ -259,7 +261,9 @@ const Advanced = () => {
             </Tooltip>
           </div>
           <div className="mt-2 w-full rounded-lg bg-secondary p-4">
-            <label className="mb-1 inline-block font-medium">Choose GPU</label>
+            <label className="mb-1 inline-block font-medium">
+              Choose device(s)
+            </label>
             <Select
               disabled={gpuList.length === 0 || !gpuEnabled}
               value={selectedGpu.join()}
@@ -328,6 +332,26 @@ const Advanced = () => {
         </div>
       )}
 
+      {/* Vulkan for AMD GPU/ APU and Intel Arc GPU */}
+      {/* TODO: This needs to be set based on user toggle in settings for enabled BOOLEAN and index as INT */}
+      {!isMac && (
+        <div className="flex w-full items-start justify-between border-b border-border py-4 first:pt-0 last:border-none">
+          <div className="flex-shrink-0 space-y-1.5">
+            <div className="flex gap-x-2">
+              <h6 className="text-sm font-semibold capitalize">Vulkan GPU</h6>
+            </div>
+            <p className="text-xs leading-relaxed">
+              Enable Vulkan with AMD GPU/APU and Intel Arc GPU for better model
+              performance. 0 - iGPU, n positive - eGPU
+            </p>
+          </div>
+          {vulkanEnabled && (
+            <Input value={vulkanEnabled ? 1 : 0} className="w-[60px] pr-8" />
+          )}
+          <Switch checked={vulkanEnabled} onCheckedChange={setVulkanEnabled} />
+        </div>
+      )}
+
       <DataFolder />
       {/* Proxy */}
       <div className="flex w-full items-start justify-between border-b border-border py-4 first:pt-0 last:border-none">