Cleanup

package.json

@@ -10,7 +10,7 @@
   "packageManager": "[email protected]",
   "scripts": {
     "preinstall": "node scripts/block-npm-install.js",
-    "benchmark": "pnpm --filter=n8n benchmark",
+    "benchmark": "pnpm --filter=n8n benchmark:sqlite",
     "build": "turbo run build",
     "build:backend": "pnpm --filter=!@n8n/chat --filter=!n8n-design-system --filter=!n8n-editor-ui build",
     "build:frontend": "pnpm --filter=@n8n/chat --filter=n8n-design-system --filter=n8n-editor-ui build",

packages/cli/package.json

@@ -20,7 +20,7 @@
     "bin": "n8n"
   },
   "scripts": {
-    "benchmark": "pnpm build:benchmark && NODE_ENV=benchmark node dist/benchmark/main.js",
+    "benchmark:sqlite": "pnpm build:benchmark && NODE_ENV=benchmark node dist/benchmark/main.js",
     "clean": "rimraf dist .turbo",
     "typecheck": "tsc",
     "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json && node scripts/build.mjs",

packages/cli/src/benchmark/benchmarks.md

@@ -0,0 +1,21 @@
+# Benchmarks
+
+To run benchmarks locally in `cli`:
+
+```sh
+pnpm benchmark:sqlite
+```
+
+## Creating
+
+To create a benchmark, @TODO
+
+## Listing
+
+### 1. Production workflow with authless webhook node
+
+1.1. using "Respond immediately" mode
+1.2. using "When last node finishes" mode
+1.3. using "Using 'Respond to Webhook' node" mode
+
+All workflows with default settings.

packages/cli/src/benchmark/lib/errors/unsupported-database.error.ts

@@ -0,0 +1,7 @@
+import { ApplicationError } from 'n8n-workflow';
+
+export class UnsupportedDatabaseError extends ApplicationError {
+	constructor() {
+		super('Currently only sqlite is supported for benchmarking', { level: 'warning' });
+	}
+}

packages/cli/src/benchmark/lib/global-hooks.ts → packages/cli/src/benchmark/lib/hooks.ts

@@ -13,7 +13,11 @@ import { jsonParse } from 'n8n-workflow';
 import { readFile } from 'fs/promises';
 import type { WorkflowRequest } from '@/workflows/workflow.request';
 import { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner';
+import { Logger } from '@/Logger';
 
+/**
+ * Create a temp `.n8n` dir for encryption key, sqlite DB, etc.
+ */
 function n8nDir() {
 	const baseDirPath = path.join(tmpdir(), 'n8n-benchmarks/');
 
@@ -31,66 +35,69 @@ function n8nDir() {
 		'utf-8',
 	);
 
-	/**
-	 * @TODO Better approach than overriding? Setting N8N_USER_FOLDER has no effect
-	 */
+	// @TODO: Find better approach than overriding like this
+	// Setting N8N_USER_FOLDER has no effect
 	const instanceSettings = Container.get(InstanceSettings);
 	instanceSettings.n8nFolder = _n8nDir;
 	Container.set(InstanceSettings, instanceSettings);
 
-	console.info('.n8n dir', _n8nDir);
+	Container.get(Logger).info(`Temp .n8n dir location: ${instanceSettings.n8nFolder}`);
 }
 
-let main: Start;
-
-async function mainProcess() {
-	const args: string[] = [];
-	const _config = new Config({ root: __dirname });
-
-	main = new Start(args, _config);
-
-	await main.init();
-	await main.run();
-}
-
-async function loadFixtures(owner: User) {
-	const files = await glob('fixtures/*.json', {
+/**
+ * Load into DB and activate in memory all workflows to use in benchmarks.
+ */
+async function prepareWorkflows(owner: User) {
+	const files = await glob('workflows/*.json', {
 		cwd: path.join('dist', 'benchmark'),
 		absolute: true,
 	});
 
-	const fixtures: WorkflowRequest.CreatePayload[] = [];
+	const workflows: WorkflowRequest.CreatePayload[] = [];
 
 	for (const file of files) {
 		const content = await readFile(file, 'utf8');
-		fixtures.push(jsonParse<WorkflowRequest.CreatePayload>(content));
+		workflows.push(jsonParse<WorkflowRequest.CreatePayload>(content));
 	}
 
-	for (const fixture of fixtures) {
-		try {
-			// @ts-ignore @TODO Fix typing
-			await Container.get(WorkflowsController).create({ body: fixture, user: owner });
-			await Container.get(ActiveWorkflowRunner).add(fixture.id as string, 'activate');
-		} catch (e) {
-			console.log(e);
-		}
+	for (const workflow of workflows) {
+		// @ts-ignore @TODO Fix typing
+		await Container.get(WorkflowsController).create({ body: workflow, user: owner });
+		await Container.get(ActiveWorkflowRunner).add(workflow.id as string, 'activate');
 	}
+}
+
+let main: Start;
 
-	// const allActive = await Container.get(WorkflowRepository).getAllActive();
-	// console.log('allActive', allActive);
+/**
+ * Start the main n8n process to use in benchmarks.
+ */
+async function mainProcess() {
+	const args: string[] = [];
+	const _config = new Config({ root: __dirname });
+
+	main = new Start(args, _config);
+
+	await main.init();
+	await main.run();
 }
 
-export async function setup() {
+/**
+ * Setup to run before once all benchmarks.
+ */
+export async function globalSetup() {
 	n8nDir();
 
 	await mainProcess();
-	// @TODO: Postgres?
 
 	const owner = await Container.get(UserRepository).createTestOwner();
 
-	await loadFixtures(owner);
+	await prepareWorkflows(owner);
 }
 
-export async function teardown() {
+/**
+ * Teardown to run before after all benchmarks.
+ */
+export async function globalTeardown() {
 	await main.stopProcess();
 }

packages/cli/src/benchmark/lib/suites.ts

@@ -5,7 +5,7 @@ import { assert } from 'n8n-workflow';
 import glob from 'fast-glob';
 import callsites from 'callsites';
 import type { Suites, Task, Callback } from './types';
-import { DuplicateHookError } from './duplicate-hook.error';
+import { DuplicateHookError } from './errors/duplicate-hook.error';
 
 export const suites: Suites = {};
 
@@ -33,7 +33,7 @@ export function registerSuites(bench: Bench) {
 		 * In jest and vitest, `beforeAll` and `afterAll` refer to all tests in a suite,
 		 * while `beforeEach` and `afterEach` refer to each individual test.
 		 *
-		 * The API renames tinybench's hooks to prevent confusion from this difference.
+		 * We rename tinybench's hooks to prevent confusion from this difference.
 		 */
 		const options: Record<string, Callback> = {};
 
@@ -58,16 +58,19 @@ function suiteFilePath() {
 }
 
 /**
- * Benchmarking API
+ * Run a benchmarking task, i.e. a single operation whose performance to measure.
  */
-
 export function task(description: string, operation: Task['operation']) {
 	const filePath = suiteFilePath();
 
 	suites[filePath] ||= { hooks: {}, tasks: [] };
 	suites[filePath].tasks.push({ description, operation });
 }
 
+/**
+ * Setup step to run once before each benchmarking task in a suite.
+ * Only one `beforeEach` is allowed per suite.
+ */
 export function beforeEach(fn: Callback) {
 	const filePath = suiteFilePath();
 
@@ -79,6 +82,10 @@ export function beforeEach(fn: Callback) {
 	suites[filePath].hooks.beforeEach = fn;
 }
 
+/**
+ * Teardown step to run once after each benchmarking task in a suite.
+ * Only one `afterEach` is allowed per suite.
+ */
 export function afterEach(fn: Callback) {
 	const filePath = suiteFilePath();
 

packages/cli/src/benchmark/lib/types.ts

@@ -1,3 +1,6 @@
+/**
+ * Benchmarking suites, i.e. `*.tasks.ts` files containing benchmarking tasks.
+ */
 export type Suites = {
 	[suiteFilepath: string]: {
 		hooks: {
@@ -8,7 +11,9 @@ export type Suites = {
 	};
 };
 
-/** A benchmarking task, i.e. a single operation whose performance to measure. */
+/**
+ * A benchmarking task, i.e. a single operation whose performance to measure.
+ */
 export type Task = {
 	description: string;
 	operation: Callback;

packages/cli/src/benchmark/main.ts

@@ -1,6 +1,10 @@
 import 'reflect-metadata';
-import * as globalHooks from './lib/global-hooks';
+import * as hooks from './lib/hooks';
 import { collectSuites, registerSuites, suiteCount } from './lib/suites';
+import config from '@/config';
+import { UnsupportedDatabaseError } from './lib/errors/unsupported-database.error';
+import { Logger } from '@/Logger';
+import Container from 'typedi';
 
 /* eslint-disable import/no-extraneous-dependencies */
 import Bench from 'tinybench';
@@ -10,16 +14,24 @@ import { withCodSpeed } from '@codspeed/tinybench-plugin';
 export { beforeEach, afterEach, task } from './lib/suites';
 
 async function main() {
+	const dbType = config.getEnv('database.type');
+
+	if (dbType !== 'sqlite') throw new UnsupportedDatabaseError();
+
 	await collectSuites();
 
 	const count = suiteCount();
 
+	const logger = Container.get(Logger);
+
 	if (count === 0) {
-		console.log('No benchmarking suites found');
+		logger.info('No benchmarking suites found. Exiting...');
 		return;
 	}
 
-	await globalHooks.setup();
+	logger.info(`Running ${count} benchmarking ${count === 1 ? 'suite' : 'suites'}...`);
+
+	await hooks.globalSetup();
 
 	const _bench = new Bench({
 		time: 0, // @TODO: Temp value
@@ -30,13 +42,11 @@ async function main() {
 
 	registerSuites(bench);
 
-	console.log(`Running ${count} benchmarking suites...`);
-
 	await bench.run();
 
-	console.table(bench.table());
+	console.table(bench.table()); // @TODO: Output properly? Ref. Codspeed
 
-	await globalHooks.teardown();
+	await hooks.globalTeardown();
 }
 
 void main();

packages/cli/tsconfig.benchmark.json

@@ -5,6 +5,6 @@
 		"outDir": "dist",
 		"tsBuildInfoFile": "dist/benchmark.tsbuildinfo"
 	},
-	"include": ["src/**/*.ts", "src/benchmark/**/*.ts", "src/benchmark/fixtures/*.json"],
+	"include": ["src/**/*.ts", "src/benchmark/**/*.ts", "src/benchmark/workflows/*.json"],
 	"exclude": ["test/**"]
 }