fixes: #247 - inference plugin should check nitro service available (#…

…313)

* fix: #247 - inference plugin should check nitro service available

* fix: #247 check service status and emit error if any

* chore: error handling

* chore: typo

* fix: open conversation does not work when model is deleted

* chore: reload plugins in development mode without exiting the process

* chore: move model file check to inference plugin

* update package-lock.json

---------

Co-authored-by: Hien To <>

electron/core/plugins/data-plugin/module.ts

@@ -129,6 +129,7 @@ function updateFinishedDownloadAt(fileName: string, time: number) {
     db.run(stmt, [time, fileName], (err: any) => {
       if (err) {
         console.log(err);
+        res(undefined);
       } else {
         console.log("Updated 1 row");
         res("Updated");
@@ -150,7 +151,11 @@ function getUnfinishedDownloadModels() {
 
     const query = `SELECT * FROM models WHERE finish_download_at = -1 ORDER BY start_download_at DESC`;
     db.all(query, (err: Error, row: any) => {
-      res(row);
+      if (row) {
+        res(row);
+      } else {
+        res([]);
+      }
     });
     db.close();
   });
@@ -193,13 +198,10 @@ function getModelById(modelId: string) {
       path.join(app.getPath("userData"), "jan.db")
     );
 
-    console.debug("Get model by id", modelId);
     db.get(
       `SELECT * FROM models WHERE id = ?`,
       [modelId],
       (err: any, row: any) => {
-        console.debug("Get model by id result", row);
-
         if (row) {
           const product = {
             id: row.id,
@@ -223,6 +225,8 @@ function getModelById(modelId: string) {
             downloadUrl: row.download_url,
           };
           res(product);
+        } else {
+          res(undefined);
         }
       }
     );

electron/core/plugins/inference-plugin/index.js → electron/core/plugins/inference-plugin/index.ts

@@ -9,14 +9,6 @@ const initModel = async (product) =>
     }
   });
 
-const dispose = async () =>
-  new Promise(async (resolve) => {
-    if (window.electronAPI) {
-      window.electronAPI
-        .invokePluginFunc(MODULE_PATH, "dispose")
-        .then((res) => resolve(res));
-    }
-  });
 const inferenceUrl = () => "http://localhost:3928/llama/chat_completion";
 
 const stopModel = () => {
@@ -27,6 +19,5 @@ const stopModel = () => {
 export function init({ register }) {
   register("initModel", "initModel", initModel);
   register("inferenceUrl", "inferenceUrl", inferenceUrl);
-  register("dispose", "dispose", dispose);
   register("stopModel", "stopModel", stopModel);
 }

electron/core/plugins/inference-plugin/module.ts

@@ -0,0 +1,119 @@
+const path = require("path");
+const { app } = require("electron");
+const { spawn } = require("child_process");
+const fs = require("fs");
+const tcpPortUsed = require("tcp-port-used");
+const { killPortProcess } = require("kill-port-process");
+
+let subprocess = null;
+const PORT = 3928;
+
+const initModel = (product) => {
+  return (
+    new Promise<void>(async (resolve, reject) => {
+      if (!product?.fileName) {
+        reject("Model not found, please download again.");
+      }
+      if (subprocess) {
+        console.error(
+          "A subprocess is already running. Attempt to kill then reinit."
+        );
+        killSubprocess();
+      }
+      resolve(product?.fileName);
+    })
+      // Kill port process if it is already in use
+      .then((fileName) =>
+        tcpPortUsed
+          .waitUntilFree(PORT, 200, 3000)
+          .catch(() => killPortProcess(PORT))
+          .then(() => fileName)
+      )
+      // Spawn Nitro subprocess to load model
+      .then(() => {
+        let binaryFolder = path.join(__dirname, "nitro"); // Current directory by default
+
+        // Read the existing config
+        const configFilePath = path.join(binaryFolder, "config", "config.json");
+        let config: any = {};
+        if (fs.existsSync(configFilePath)) {
+          const rawData = fs.readFileSync(configFilePath, "utf-8");
+          config = JSON.parse(rawData);
+        }
+
+        // Update the llama_model_path
+        if (!config.custom_config) {
+          config.custom_config = {};
+        }
+
+        const modelPath = path.join(app.getPath("userData"), product.fileName);
+
+        config.custom_config.llama_model_path = modelPath;
+
+        // Write the updated config back to the file
+        fs.writeFileSync(configFilePath, JSON.stringify(config, null, 4));
+
+        let binaryName;
+
+        if (process.platform === "win32") {
+          binaryName = "nitro_windows_amd64.exe";
+        } else if (process.platform === "darwin") {
+          // Mac OS platform
+          binaryName =
+            process.arch === "arm64" ? "nitro_mac_arm64" : "nitro_mac_amd64";
+        } else {
+          // Linux
+          binaryName = "nitro_linux_amd64_cuda"; // For other platforms
+        }
+
+        const binaryPath = path.join(binaryFolder, binaryName);
+
+        // Execute the binary
+
+        subprocess = spawn(binaryPath, [configFilePath], { cwd: binaryFolder });
+
+        // Handle subprocess output
+        subprocess.stdout.on("data", (data) => {
+          console.log(`stdout: ${data}`);
+        });
+
+        subprocess.stderr.on("data", (data) => {
+          console.error(`stderr: ${data}`);
+        });
+
+        subprocess.on("close", (code) => {
+          console.log(`child process exited with code ${code}`);
+          subprocess = null;
+        });
+      })
+      .then(() => tcpPortUsed.waitUntilUsed(PORT, 300, 30000))
+      .then(() => {
+        return {};
+      })
+      .catch((err) => {
+        return { error: err };
+      })
+  );
+};
+
+function dispose() {
+  killSubprocess();
+  // clean other registered resources here
+}
+
+function killSubprocess() {
+  if (subprocess) {
+    subprocess.kill();
+    subprocess = null;
+    console.log("Subprocess terminated.");
+  } else {
+    killPortProcess(PORT);
+    console.error("No subprocess is currently running.");
+  }
+}
+
+module.exports = {
+  initModel,
+  killSubprocess,
+  dispose,
+};