chore: add model.json for Llama3 and other outdated model version (#2773

) * chore: add model.json for Llama3 and other outdated model version * fix: consistency format * fix: correct folder id * update: bump version * add: stop words * fix: model.json * Update extensions/inference-nitro-extension/resources/models/llama3-8b-instruct/model.json * Update extensions/inference-nitro-extension/resources/models/llama3-8b-instruct/model.json Based on suggested change Co-authored-by: Nikolaus Kühn <[email protected]> --------- Co-authored-by: Van-QA <[email protected]> Co-authored-by: Hoang Ha <[email protected]> Co-authored-by: Louis <[email protected]> Co-authored-by: Nikolaus Kühn <[email protected]>
janhq · Apr 22, 2024 · 67db45f · 67db45f
1 parent 83e9e2b
commit 67db45f
Show file tree

Hide file tree

Showing 5 changed files with 39 additions and 3 deletions.
diff --git a/extensions/inference-nitro-extension/package.json b/extensions/inference-nitro-extension/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@janhq/inference-nitro-extension",
   "productName": "Nitro Inference Engine",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "description": "This extension embeds Nitro, a lightweight (3mb) inference engine written in C++. See https://nitro.jan.ai.\nAdditional dependencies could be installed to run without Cuda Toolkit installation.",
   "main": "dist/index.js",
   "node": "dist/node/index.cjs.js",

diff --git a/extensions/inference-nitro-extension/resources/models/command-r-34b/model.json b/extensions/inference-nitro-extension/resources/models/command-r-34b/model.json
@@ -8,7 +8,7 @@
     "id": "command-r-34b",
     "object": "model",
     "name": "Command-R v01 34B Q4",
-    "version": "1.1",
+    "version": "1.2",
     "description": "C4AI Command-R developed by CohereAI is optimized for a variety of use cases including reasoning, summarization, and question answering.",
     "format": "gguf",
     "settings": {

diff --git a/extensions/inference-nitro-extension/resources/models/llama3-8b-instruct/model.json b/extensions/inference-nitro-extension/resources/models/llama3-8b-instruct/model.json
@@ -0,0 +1,34 @@
+{
+    "sources": [
+      {
+        "filename": "Meta-Llama-3-8B-Instruct-Q4_K_M.gguf",
+        "url": "https://huggingface.co/lmstudio-community/Meta-Llama-3-8B-Instruct-GGUF/resolve/main/Meta-Llama-3-8B-Instruct-Q4_K_M.gguf"
+      }
+    ],
+    "id": "llama3-8b-instruct",
+    "object": "model",
+    "name": "Llama 3 8B Q4",
+    "version": "1.0",
+    "description": "Meta's Llama 3 excels at general usage situations, including chat, general world knowledge, and coding.",
+    "format": "gguf",
+    "settings": {
+      "ctx_len": 8192,
+      "prompt_template": "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n{system_message}<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n{prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
+      "llama_model_path": "Meta-Llama-3-8B-Instruct-Q4_K_M.gguf"
+    },
+    "parameters": {
+      "temperature": 0.7,
+      "top_p": 0.95,
+      "stream": true,
+      "max_tokens": 4096,
+      "stop": ["<|end_of_text|>","<|eot_id|>"],
+      "frequency_penalty": 0,
+      "presence_penalty": 0
+    },
+    "metadata": {
+      "author": "MetaAI",
+      "tags": ["7B", "Featured"],
+      "size": 4920000000
+    },
+    "engine": "nitro"
+}
diff --git a/extensions/inference-nitro-extension/resources/models/phind-34b/model.json b/extensions/inference-nitro-extension/resources/models/phind-34b/model.json
@@ -8,7 +8,7 @@
   "id": "phind-34b",
   "object": "model",
   "name": "Phind 34B Q4",
-  "version": "1.0",
+  "version": "1.1",
   "description": "Phind 34B is the best Open-source coding model.",
   "format": "gguf",
   "settings": {

diff --git a/extensions/inference-nitro-extension/rollup.config.ts b/extensions/inference-nitro-extension/rollup.config.ts
@@ -36,6 +36,7 @@ const trinityv127bJson = require('./resources/models/trinity-v1.2-7b/model.json'
 const vistral7bJson = require('./resources/models/vistral-7b/model.json')
 const wizardcoder13bJson = require('./resources/models/wizardcoder-13b/model.json')
 const yi34bJson = require('./resources/models/yi-34b/model.json')
+const llama3Json = require('./resources/models/llama3-8b-instruct/model.json')
 
 export default [
   {
@@ -79,6 +80,7 @@ export default [
           vistral7bJson,
           wizardcoder13bJson,
           yi34bJson,
+          llama3Json
         ]),
         NODE: JSON.stringify(`${packageJson.name}/${packageJson.node}`),
         DEFAULT_SETTINGS: JSON.stringify(defaultSettingJson),