refactor: introduce node module in nitro extension

core/package.json

@@ -15,13 +15,6 @@
     "dist"
   ],
   "author": "Jan <[email protected]>",
-  "repository": {
-    "type": "git",
-    "url": ""
-  },
-  "engines": {
-    "node": ">=6.0.0"
-  },
   "exports": {
     ".": "./dist/core.umd.js",
     "./sdk": "./dist/core.umd.js",
@@ -49,53 +42,6 @@
     "build": "tsc --module commonjs && rollup -c rollup.config.ts",
     "start": "rollup -c rollup.config.ts -w"
   },
-  "lint-staged": {
-    "{src,test}/**/*.ts": [
-      "prettier --write",
-      "git add"
-    ]
-  },
-  "config": {
-    "commitizen": {
-      "path": "node_modules/cz-conventional-changelog"
-    }
-  },
-  "jest": {
-    "transform": {
-      ".(ts|tsx)": "ts-jest"
-    },
-    "testEnvironment": "node",
-    "testRegex": "(/__tests__/.*|\\.(test|spec))\\.(ts|tsx|js)$",
-    "moduleFileExtensions": [
-      "ts",
-      "tsx",
-      "js"
-    ],
-    "coveragePathIgnorePatterns": [
-      "/node_modules/",
-      "/test/"
-    ],
-    "coverageThreshold": {
-      "global": {
-        "branches": 90,
-        "functions": 95,
-        "lines": 95,
-        "statements": 95
-      }
-    },
-    "collectCoverageFrom": [
-      "src/*.{js,ts}"
-    ]
-  },
-  "prettier": {
-    "semi": false,
-    "singleQuote": true
-  },
-  "commitlint": {
-    "extends": [
-      "@commitlint/config-conventional"
-    ]
-  },
   "devDependencies": {
     "@types/node": "^12.0.2",
     "rollup": "^2.38.5",
@@ -104,7 +50,6 @@
     "rollup-plugin-node-resolve": "^5.2.0",
     "rollup-plugin-sourcemaps": "^0.6.3",
     "rollup-plugin-typescript2": "^0.36.0",
-    "ts-node": "^7.0.1",
     "tslib": "^2.6.2",
     "typescript": "^5.2.2"
   }

core/src/types/model/modelEntity.ts

@@ -104,6 +104,9 @@ export type ModelSettingParams = {
   n_parallel?: number
   cpu_threads?: number
   prompt_template?: string
+  system_prompt?: string
+  ai_prompt?: string
+  user_prompt?: string
 }
 
 /**

extensions/inference-nitro-extension/package.json

@@ -3,11 +3,11 @@
   "version": "1.0.0",
   "description": "This extension embeds Nitro, a lightweight (3mb) inference engine written in C++. See nitro.jan.ai",
   "main": "dist/index.js",
-  "module": "dist/module.js",
+  "node": "dist/node/index.cjs.js",
   "author": "Jan <[email protected]>",
   "license": "AGPL-3.0",
   "scripts": {
-    "build": "tsc -b . && webpack --config webpack.config.js",
+    "build": "tsc --module commonjs && rollup -c rollup.config.ts",
     "downloadnitro:linux": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64.tar.gz -e --strip 1 -o ./bin/linux-cpu && chmod +x ./bin/linux-cpu/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/linux-cuda-12-0 && chmod +x ./bin/linux-cuda-12-0/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/linux-cuda-11-7 && chmod +x ./bin/linux-cuda-11-7/nitro",
     "downloadnitro:darwin": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-arm64.tar.gz -e --strip 1 -o ./bin/mac-arm64 && chmod +x ./bin/mac-arm64/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-amd64.tar.gz -e --strip 1 -o ./bin/mac-x64 && chmod +x ./bin/mac-x64/nitro",
     "downloadnitro:win32": "download.bat",
@@ -19,24 +19,33 @@
   },
   "exports": {
     ".": "./dist/index.js",
-    "./main": "./dist/module.js"
+    "./main": "./dist/node/index.cjs.js"
   },
   "devDependencies": {
+    "@rollup/plugin-commonjs": "^25.0.7",
+    "@rollup/plugin-json": "^6.1.0",
+    "@rollup/plugin-node-resolve": "^15.2.3",
+    "@types/node": "^20.11.4",
+    "@types/tcp-port-used": "^1.0.4",
     "cpx": "^1.5.0",
+    "download-cli": "^1.1.1",
     "rimraf": "^3.0.2",
+    "rollup": "^2.38.5",
+    "rollup-plugin-define": "^1.0.1",
+    "rollup-plugin-sourcemaps": "^0.6.3",
+    "rollup-plugin-typescript2": "^0.36.0",
     "run-script-os": "^1.1.6",
-    "webpack": "^5.88.2",
-    "webpack-cli": "^5.1.4"
+    "typescript": "^5.3.3"
   },
   "dependencies": {
     "@janhq/core": "file:../../core",
-    "download-cli": "^1.1.1",
+    "@rollup/plugin-replace": "^5.0.5",
+    "@types/os-utils": "^0.0.4",
     "fetch-retry": "^5.0.6",
     "os-utils": "^0.0.14",
     "path-browserify": "^1.0.1",
     "rxjs": "^7.8.1",
     "tcp-port-used": "^1.0.2",
-    "ts-loader": "^9.5.0",
     "ulid": "^2.3.0"
   },
   "engines": {

extensions/inference-nitro-extension/rollup.config.ts

@@ -0,0 +1,77 @@
+import resolve from "@rollup/plugin-node-resolve";
+import commonjs from "@rollup/plugin-commonjs";
+import sourceMaps from "rollup-plugin-sourcemaps";
+import typescript from "rollup-plugin-typescript2";
+import json from "@rollup/plugin-json";
+import replace from "@rollup/plugin-replace";
+const packageJson = require("./package.json");
+
+const pkg = require("./package.json");
+
+export default [
+  {
+    input: `src/index.ts`,
+    output: [{ file: pkg.main, format: "es", sourcemap: true }],
+    // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash')
+    external: [],
+    watch: {
+      include: "src/**",
+    },
+    plugins: [
+      replace({
+        NODE: JSON.stringify(`${packageJson.name}/${packageJson.node}`),
+        INFERENCE_URL: JSON.stringify(
+          process.env.INFERENCE_URL ||
+            "http://127.0.0.1:3928/inferences/llamacpp/chat_completion"
+        ),
+        TROUBLESHOOTING_URL: JSON.stringify(
+          "https://jan.ai/guides/troubleshooting"
+        ),
+      }),
+      // Allow json resolution
+      json(),
+      //     Compile TypeScript files
+      typescript({ useTsconfigDeclarationDir: true }),
+      // Compile TypeScript files
+      // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs)
+      commonjs(),
+      // Allow node_modules resolution, so you can use 'external' to control
+      // which external modules to include in the bundle
+      // https://github.com/rollup/rollup-plugin-node-resolve#usage
+      resolve({
+        extensions: [".js", ".ts", ".svelte"],
+      }),
+
+      // Resolve source maps to the original source
+      sourceMaps(),
+    ],
+  },
+  {
+    input: `src/node/index.ts`,
+    output: [
+      { file: "dist/node/index.cjs.js", format: "cjs", sourcemap: true },
+    ],
+    // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash')
+    external: ["@janhq/core/node"],
+    watch: {
+      include: "src/node/**",
+    },
+    plugins: [
+      // Allow json resolution
+      json(),
+      // Compile TypeScript files
+      typescript({ useTsconfigDeclarationDir: true }),
+      // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs)
+      commonjs(),
+      // Allow node_modules resolution, so you can use 'external' to control
+      // which external modules to include in the bundle
+      // https://github.com/rollup/rollup-plugin-node-resolve#usage
+      resolve({
+        extensions: [".ts", ".js", ".json"],
+      }),
+
+      // Resolve source maps to the original source
+      sourceMaps(),
+    ],
+  },
+];

extensions/inference-nitro-extension/src/@types/global.d.ts

@@ -1,4 +1,4 @@
-declare const MODULE: string;
+declare const NODE: string;
 declare const INFERENCE_URL: string;
 declare const TROUBLESHOOTING_URL: string;
 

extensions/inference-nitro-extension/src/index.ts

@@ -26,7 +26,6 @@ import {
 } from "@janhq/core";
 import { requestInference } from "./helpers/sse";
 import { ulid } from "ulid";
-import { join } from "path";
 
 /**
  * A class that implements the InferenceExtension interface from the @janhq/core package.
@@ -43,7 +42,7 @@ export default class JanInferenceNitroExtension implements InferenceExtension {
    */
   private static readonly _intervalHealthCheck = 5 * 1000;
 
-  private _currentModel: Model;
+  private _currentModel: Model | undefined;
 
   private _engineSettings: EngineSettings = {
     ctx_len: 2048,
@@ -82,15 +81,17 @@ export default class JanInferenceNitroExtension implements InferenceExtension {
     if (!(await fs.existsSync(JanInferenceNitroExtension._homeDir))) {
       await fs
         .mkdirSync(JanInferenceNitroExtension._homeDir)
-        .catch((err) => console.debug(err));
+        .catch((err: Error) => console.debug(err));
     }
 
     if (!(await fs.existsSync(JanInferenceNitroExtension._settingsDir)))
       await fs.mkdirSync(JanInferenceNitroExtension._settingsDir);
     this.writeDefaultEngineSettings();
 
     // Events subscription
-    events.on(EventName.OnMessageSent, (data) => this.onMessageRequest(data));
+    events.on(EventName.OnMessageSent, (data: MessageRequest) =>
+      this.onMessageRequest(data)
+    );
 
     events.on(EventName.OnModelInit, (model: Model) => this.onModelInit(model));
 
@@ -99,7 +100,7 @@ export default class JanInferenceNitroExtension implements InferenceExtension {
     events.on(EventName.OnInferenceStopped, () => this.onInferenceStopped());
 
     // Attempt to fetch nvidia info
-    await executeOnMain(MODULE, "updateNvidiaInfo", {});
+    await executeOnMain(NODE, "updateNvidiaInfo", {});
   }
 
   /**
@@ -109,10 +110,10 @@ export default class JanInferenceNitroExtension implements InferenceExtension {
 
   private async writeDefaultEngineSettings() {
     try {
-      const engineFile = join(
+      const engineFile = await joinPath([
         JanInferenceNitroExtension._homeDir,
-        JanInferenceNitroExtension._engineMetadataFileName
-      );
+        JanInferenceNitroExtension._engineMetadataFileName,
+      ]);
       if (await fs.existsSync(engineFile)) {
         const engine = await fs.readFileSync(engineFile, "utf-8");
         this._engineSettings =
@@ -133,12 +134,12 @@ export default class JanInferenceNitroExtension implements InferenceExtension {
 
     const modelFullPath = await joinPath(["models", model.id]);
 
-    const nitroInitResult = await executeOnMain(MODULE, "initModel", {
-      modelFullPath: modelFullPath,
-      model: model,
+    const nitroInitResult = await executeOnMain(NODE, "runModel", {
+      modelFullPath,
+      model,
     });
 
-    if (nitroInitResult.error === null) {
+    if (nitroInitResult?.error) {
       events.emit(EventName.OnModelFail, model);
       return;
     }
@@ -155,12 +156,11 @@ export default class JanInferenceNitroExtension implements InferenceExtension {
   private async onModelStop(model: Model) {
     if (model.engine !== "nitro") return;
 
-    await executeOnMain(MODULE, "stopModel");
+    await executeOnMain(NODE, "stopModel");
     events.emit(EventName.OnModelStopped, {});
 
     // stop the periocally health check
     if (this.getNitroProcesHealthIntervalId) {
-      console.debug("Stop calling Nitro process health check");
       clearInterval(this.getNitroProcesHealthIntervalId);
       this.getNitroProcesHealthIntervalId = undefined;
     }
@@ -170,7 +170,7 @@ export default class JanInferenceNitroExtension implements InferenceExtension {
    * Periodically check for nitro process's health.
    */
   private async periodicallyGetNitroHealth(): Promise<void> {
-    const health = await executeOnMain(MODULE, "getCurrentNitroProcessInfo");
+    const health = await executeOnMain(NODE, "getCurrentNitroProcessInfo");
 
     const isRunning = this.nitroProcessInfo?.isRunning ?? false;
     if (isRunning && health.isRunning === false) {
@@ -204,6 +204,8 @@ export default class JanInferenceNitroExtension implements InferenceExtension {
     };
 
     return new Promise(async (resolve, reject) => {
+      if (!this._currentModel) return Promise.reject("No model loaded");
+
       requestInference(data.messages ?? [], this._currentModel).subscribe({
         next: (_content) => {},
         complete: async () => {
@@ -223,7 +225,9 @@ export default class JanInferenceNitroExtension implements InferenceExtension {
    * @param {MessageRequest} data - The data for the new message request.
    */
   private async onMessageRequest(data: MessageRequest) {
-    if (data.model.engine !== "nitro") return;
+    if (data.model?.engine !== InferenceEngine.nitro || !this._currentModel) {
+      return;
+    }
 
     const timestamp = Date.now();
     const message: ThreadMessage = {
@@ -242,11 +246,12 @@ export default class JanInferenceNitroExtension implements InferenceExtension {
     this.isCancelled = false;
     this.controller = new AbortController();
 
-    requestInference(
-      data.messages ?? [],
-      { ...this._currentModel, ...data.model },
-      this.controller
-    ).subscribe({
+    // @ts-ignore
+    const model: Model = {
+      ...(this._currentModel || {}),
+      ...(data.model || {}),
+    };
+    requestInference(data.messages ?? [], model, this.controller).subscribe({
       next: (content) => {
         const messageContent: ThreadContent = {
           type: ContentType.Text,