Skip to content

Commit f0361f3

Browse files
giladgdngxson
andauthored
local-apps: update node-llama-cpp snippet (#1169)
This PR updates the code snippet of `node-llama-cpp` to use a tag-based model URI, like in `llama.cpp` and Ollama. The implementation for this uses the Ollama support on Hugging Face, and is based on this PR: ggml-org/llama.cpp#11195 Examples: ```bash # chat with the model npx -y node-llama-cpp chat hf:mradermacher/Llama-3.2-3B-Instruct-GGUF:Q4_K_M # estimate the model compatibility with the current machine hardware npx -y node-llama-cpp inspect estimate hf:mradermacher/Llama-3.2-3B-Instruct-GGUF:Q4_K_M ``` Co-authored-by: Xuan-Son Nguyen <[email protected]>
1 parent 34edae3 commit f0361f3

File tree

1 file changed

+9
-6
lines changed

1 file changed

+9
-6
lines changed

packages/tasks/src/local-apps.ts

+9-6
Original file line numberDiff line numberDiff line change
@@ -138,18 +138,21 @@ const snippetLlamacpp = (model: ModelData, filepath?: string): LocalAppSnippet[]
138138
};
139139

140140
const snippetNodeLlamaCppCli = (model: ModelData, filepath?: string): LocalAppSnippet[] => {
141+
let tagName = "{{OLLAMA_TAG}}";
142+
143+
if (filepath) {
144+
const quantLabel = parseGGUFQuantLabel(filepath);
145+
tagName = quantLabel ? `:${quantLabel}` : tagName;
146+
}
147+
141148
return [
142149
{
143150
title: "Chat with the model",
144-
content: [
145-
`npx -y node-llama-cpp chat \\`,
146-
` --model "hf:${model.id}/${filepath ?? "{{GGUF_FILE}}"}" \\`,
147-
` --prompt 'Hi there!'`,
148-
].join("\n"),
151+
content: `npx -y node-llama-cpp chat hf:${model.id}${tagName}`,
149152
},
150153
{
151154
title: "Estimate the model compatibility with your hardware",
152-
content: `npx -y node-llama-cpp inspect estimate "hf:${model.id}/${filepath ?? "{{GGUF_FILE}}"}"`,
155+
content: `npx -y node-llama-cpp inspect estimate hf:${model.id}${tagName}`,
153156
},
154157
];
155158
};

0 commit comments

Comments
 (0)