Merge pull request #4408 from janhq/main

Sync release 0.5.13 into dev
janhq · Jan 6, 2025 · b4d2176 · b4d2176
2 parents 3f2d2ef + e3d44c9
commit b4d2176
Show file tree

Hide file tree

Showing 5 changed files with 23 additions and 3 deletions.
diff --git a/extensions/inference-cortex-extension/bin/version.txt b/extensions/inference-cortex-extension/bin/version.txt
@@ -1 +1 @@
-1.0.7
+1.0.8
diff --git a/extensions/inference-cortex-extension/resources/default_settings.json b/extensions/inference-cortex-extension/resources/default_settings.json
@@ -18,6 +18,16 @@
       "placeholder": "4"
     }
   },
+  {
+    "key": "cpu_threads",
+    "title": "CPU Threads",
+    "description": "The number of CPU threads to use (when in CPU mode)",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": "",
+      "placeholder": "Number of CPU threads"
+    }
+  },
   {
     "key": "flash_attn",
     "title": "Flash Attention enabled",

diff --git a/extensions/inference-cortex-extension/src/index.ts b/extensions/inference-cortex-extension/src/index.ts
@@ -42,6 +42,7 @@ export enum Settings {
   flash_attn = 'flash_attn',
   cache_type = 'cache_type',
   use_mmap = 'use_mmap',
+  cpu_threads = 'cpu_threads',
 }
 
 /**
@@ -65,6 +66,7 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
   flash_attn: boolean = true
   use_mmap: boolean = true
   cache_type: string = 'f16'
+  cpu_threads?: number
 
   /**
    * The URL for making inference requests.
@@ -104,6 +106,10 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
     this.flash_attn = await this.getSetting<boolean>(Settings.flash_attn, true)
     this.use_mmap = await this.getSetting<boolean>(Settings.use_mmap, true)
     this.cache_type = await this.getSetting<string>(Settings.cache_type, 'f16')
+    const threads_number = Number(
+      await this.getSetting<string>(Settings.cpu_threads, '')
+    )
+    if (!Number.isNaN(threads_number)) this.cpu_threads = threads_number
 
     this.queue.add(() => this.clean())
 
@@ -139,6 +145,9 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
       this.cache_type = value as string
     } else if (key === Settings.use_mmap && typeof value === 'boolean') {
       this.use_mmap = value as boolean
+    } else if (key === Settings.cpu_threads && typeof value === 'string') {
+      const threads_number = Number(value)
+      if (!Number.isNaN(threads_number)) this.cpu_threads = threads_number
     }
   }
 
@@ -196,6 +205,7 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
             flash_attn: this.flash_attn,
             cache_type: this.cache_type,
             use_mmap: this.use_mmap,
+            ...(this.cpu_threads ? { cpu_threads: this.cpu_threads } : {}),
           },
           timeout: false,
           signal,

diff --git a/web/hooks/useCreateNewThread.ts b/web/hooks/useCreateNewThread.ts
@@ -98,7 +98,7 @@ export const useCreateNewThread = () => {
     // Use ctx length by default
     const overriddenParameters = {
       max_tokens: !isLocalEngine(defaultModel?.engine)
-        ? (defaultModel?.parameters.token_limit ?? 8192)
+        ? (defaultModel?.parameters.max_tokens ?? 8192)
         : defaultContextLength,
     }
 

diff --git a/web/screens/Settings/Advanced/index.tsx b/web/screens/Settings/Advanced/index.tsx
@@ -417,7 +417,7 @@ const Advanced = () => {
         )}
 
         {/* Vulkan for AMD GPU/ APU and Intel Arc GPU */}
-        {!isMac && gpuList.length > 0 && experimentalEnabled && (
+        {!isMac && experimentalEnabled && (
           <div className="flex w-full flex-col items-start justify-between gap-4 border-b border-[hsla(var(--app-border))] py-4 first:pt-0 last:border-none sm:flex-row">
             <div className="space-y-1">
               <div className="flex gap-x-2">