From c43407bc66911d1320da0584261468a00c137606 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Fri, 18 Aug 2023 13:36:40 +0200 Subject: [PATCH 01/40] Reorder --- lib/result.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/result.js b/lib/result.js index 8db2da9..d0b936a 100644 --- a/lib/result.js +++ b/lib/result.js @@ -5,7 +5,7 @@ */ export class Result { constructor(options, latency) { - // options + // configuration this.url = options.url this.maxRequests = options.maxRequests this.maxSeconds = options.maxSeconds @@ -14,11 +14,11 @@ export class Result { this.requestsPerSecond = options.requestsPerSecond // results this.elapsedSeconds = latency.getElapsed(latency.initialTime) / 1000 - const meanTime = latency.totalTime / latency.totalRequests this.totalRequests = latency.totalRequests this.totalErrors = latency.totalErrors this.totalTimeSeconds = this.elapsedSeconds this.rps = Math.round(latency.totalRequests / this.elapsedSeconds) + const meanTime = latency.totalTime / latency.totalRequests this.meanLatencyMs = Math.round(meanTime * 10) / 10 this.maxLatencyMs = latency.maxLatencyMs this.minLatencyMs = latency.minLatencyMs From fa7b950300a8499d52bdba64de4a23ddeca1deb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Fri, 18 Aug 2023 13:41:44 +0200 Subject: [PATCH 02/40] Separate derived results --- lib/result.js | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/result.js b/lib/result.js index d0b936a..b09d156 100644 --- a/lib/result.js +++ b/lib/result.js @@ -12,18 +12,23 @@ export class Result { this.concurrency = options.concurrency this.agent = options.agentKeepAlive ? 'keepalive' : 'none'; this.requestsPerSecond = options.requestsPerSecond - // results + // result this.elapsedSeconds = latency.getElapsed(latency.initialTime) / 1000 this.totalRequests = latency.totalRequests this.totalErrors = latency.totalErrors this.totalTimeSeconds = this.elapsedSeconds - this.rps = Math.round(latency.totalRequests / this.elapsedSeconds) - const meanTime = latency.totalTime / latency.totalRequests - this.meanLatencyMs = Math.round(meanTime * 10) / 10 + this.accumulatedMs = latency.totalTime this.maxLatencyMs = latency.maxLatencyMs this.minLatencyMs = latency.minLatencyMs this.percentiles = latency.computePercentiles() this.errorCodes = latency.errorCodes + this.computeDerived() + } + + computeDerived() { + const meanTime = this.accumulatedMs / this.totalRequests + this.meanLatencyMs = Math.round(meanTime * 10) / 10 + this.rps = Math.round(this.totalRequests / this.elapsedSeconds) } /** From e6a6bf515f20ecedcede9af877ceac7153dcfa5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Fri, 18 Aug 2023 13:48:17 +0200 Subject: [PATCH 03/40] Move compute percentiles from latency to result --- lib/latency.js | 28 ---------------------------- lib/result.js | 27 ++++++++++++++++++++++++++- 2 files changed, 26 insertions(+), 29 deletions(-) diff --git a/lib/latency.js b/lib/latency.js index 3f913a7..64745ab 100644 --- a/lib/latency.js +++ b/lib/latency.js @@ -178,34 +178,6 @@ export class Latency { return result } - /** - * Compute the percentiles. - */ - computePercentiles() { - const percentiles = { - 50: false, - 90: false, - 95: false, - 99: false - }; - let counted = 0; - - for (let ms = 0; ms <= this.maxLatencyMs; ms++) { - if (!this.histogramMs[ms]) { - continue; - } - counted += this.histogramMs[ms]; - const percent = counted / this.totalRequests * 100; - - Object.keys(percentiles).forEach(percentile => { - if (!percentiles[percentile] && percent > percentile) { - percentiles[percentile] = ms; - } - }); - } - return percentiles; - } - /** * Show final result. */ diff --git a/lib/result.js b/lib/result.js index b09d156..52d72ee 100644 --- a/lib/result.js +++ b/lib/result.js @@ -20,8 +20,8 @@ export class Result { this.accumulatedMs = latency.totalTime this.maxLatencyMs = latency.maxLatencyMs this.minLatencyMs = latency.minLatencyMs - this.percentiles = latency.computePercentiles() this.errorCodes = latency.errorCodes + this.histogramMs = latency.histogramMs this.computeDerived() } @@ -29,6 +29,31 @@ export class Result { const meanTime = this.accumulatedMs / this.totalRequests this.meanLatencyMs = Math.round(meanTime * 10) / 10 this.rps = Math.round(this.totalRequests / this.elapsedSeconds) + this.computePercentiles() + } + + computePercentiles() { + this.percentiles = { + 50: false, + 90: false, + 95: false, + 99: false + }; + let counted = 0; + + for (let ms = 0; ms <= this.maxLatencyMs; ms++) { + if (!this.histogramMs[ms]) { + continue; + } + counted += this.histogramMs[ms]; + const percent = counted / this.totalRequests * 100; + + Object.keys(this.percentiles).forEach(percentile => { + if (!this.percentiles[percentile] && percent > percentile) { + this.percentiles[percentile] = ms; + } + }); + } } /** From 134311d242354010482b97a5f4413132f18dce6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Fri, 18 Aug 2023 14:05:48 +0200 Subject: [PATCH 04/40] Start test server in half the cores by default --- bin/testserver.js | 79 ++++++++++++++++++++++++++++++----------------- 1 file changed, 51 insertions(+), 28 deletions(-) diff --git a/bin/testserver.js b/bin/testserver.js index 95df7c3..b111306 100755 --- a/bin/testserver.js +++ b/bin/testserver.js @@ -3,40 +3,63 @@ import * as stdio from 'stdio' import {startServer} from '../lib/testserver.js' import {loadConfig} from '../lib/config.js' +import * as cluster from 'cluster' +import {cpus} from 'os' +const options = readOptions() +start(options) -const options = stdio.getopt({ - delay: {key: 'd', args: 1, description: 'Delay the response for the given milliseconds'}, - error: {key: 'e', args: 1, description: 'Return an HTTP error code'}, - percent: {key: 'p', args: 1, description: 'Return an error (default 500) only for some % of requests'}, -}); -const configuration = loadConfig() -if (options.args && options.args.length == 1) { - options.port = parseInt(options.args[0], 10); - if (!options.port) { - console.error('Invalid port'); - options.printHelp(); - process.exit(1); + +function readOptions() { + const options = stdio.getopt({ + delay: {key: 'd', args: 1, description: 'Delay the response for the given milliseconds'}, + error: {key: 'e', args: 1, description: 'Return an HTTP error code'}, + percent: {key: 'p', args: 1, description: 'Return an error (default 500) only for some % of requests'}, + cores: {key: 'c', args: 1, description: 'Number of cores to use, default is half the total'} + }); + const configuration = loadConfig() + if (options.args && options.args.length == 1) { + options.port = parseInt(options.args[0], 10); + if (!options.port) { + console.error('Invalid port'); + options.printHelp(); + process.exit(1); + } } -} -if(options.delay) { - if(isNaN(options.delay)) { - console.error('Invalid delay'); - options.printHelp(); - process.exit(1); + if(options.delay) { + if(isNaN(options.delay)) { + console.error('Invalid delay'); + options.printHelp(); + process.exit(1); + } + options.delay = parseInt(options.delay, 10); } - options.delay = parseInt(options.delay, 10); -} -if(!options.delay) { - options.delay = configuration.delay -} -if(!options.error) { - options.error = configuration.error + if(!options.delay) { + options.delay = configuration.delay + } + if(!options.error) { + options.error = configuration.error + } + if(!options.percent) { + options.percent = configuration.percent + } + if (!options.cores) { + // default is half the cores available + const totalCores = cpus().length; + options.cores = Math.round(totalCores / 2) || 1 + } + return options } -if(!options.percent) { - options.percent = configuration.percent + +function start(options) { + if (cluster.isMaster) { + for (let index = 0; index < options.cores; index++) { + cluster.fork(); + } + } else { + startServer(options); + } } -startServer(options); From 2a01a16710f22ddefed6d1a00cae6accea835152 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Fri, 18 Aug 2023 14:17:59 +0200 Subject: [PATCH 05/40] Refactor a bit --- bin/loadtest.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bin/loadtest.js b/bin/loadtest.js index 9ab8fdf..264c643 100755 --- a/bin/loadtest.js +++ b/bin/loadtest.js @@ -51,6 +51,10 @@ async function processAndRun(options) { help(); } options.url = options.args[0]; + await startTest(options) +} + +async function startTest(options) { try { const result = await loadTest(options) result.show() @@ -62,10 +66,8 @@ async function processAndRun(options) { await processAndRun(options) -/** - * Show online help. - */ function help() { options.printHelp(); process.exit(1); } + From e3ffaab91b141b59178ff16d420ee9fa96e0613a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Fri, 18 Aug 2023 14:22:55 +0200 Subject: [PATCH 06/40] Move multicore code to its own file --- bin/testserver.js | 16 +++------------- lib/multicore.js | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+), 13 deletions(-) create mode 100755 lib/multicore.js diff --git a/bin/testserver.js b/bin/testserver.js index b111306..f4d08aa 100755 --- a/bin/testserver.js +++ b/bin/testserver.js @@ -5,6 +5,7 @@ import {startServer} from '../lib/testserver.js' import {loadConfig} from '../lib/config.js' import * as cluster from 'cluster' import {cpus} from 'os' +import {getHalfCores, runTask} from '../lib/multicore.js' const options = readOptions() start(options) @@ -15,7 +16,7 @@ function readOptions() { delay: {key: 'd', args: 1, description: 'Delay the response for the given milliseconds'}, error: {key: 'e', args: 1, description: 'Return an HTTP error code'}, percent: {key: 'p', args: 1, description: 'Return an error (default 500) only for some % of requests'}, - cores: {key: 'c', args: 1, description: 'Number of cores to use, default is half the total'} + cores: {key: 'c', args: 1, description: 'Number of cores to use, default is half the total', default: getHalfCores()} }); const configuration = loadConfig() if (options.args && options.args.length == 1) { @@ -44,22 +45,11 @@ function readOptions() { if(!options.percent) { options.percent = configuration.percent } - if (!options.cores) { - // default is half the cores available - const totalCores = cpus().length; - options.cores = Math.round(totalCores / 2) || 1 - } return options } function start(options) { - if (cluster.isMaster) { - for (let index = 0; index < options.cores; index++) { - cluster.fork(); - } - } else { - startServer(options); - } + runTask(options.cores, () => startServer(options)) } diff --git a/lib/multicore.js b/lib/multicore.js new file mode 100755 index 0000000..9949e94 --- /dev/null +++ b/lib/multicore.js @@ -0,0 +1,22 @@ +#!/usr/bin/env node + +import * as cluster from 'cluster' +import {cpus} from 'os' + + +export function getHalfCores() { + const totalCores = cpus().length + return Math.round(totalCores / 2) || 1 +} + +export function runTask(cores, task) { + if (cluster.isMaster) { + for (let index = 0; index < cores; index++) { + cluster.fork() + } + } else { + task() + } +} + + From 2fec208550e44f8e6002a9f8bcc9c090829baa19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Fri, 18 Aug 2023 17:40:36 +0200 Subject: [PATCH 07/40] Run single-core if cores=1 --- lib/multicore.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/multicore.js b/lib/multicore.js index 9949e94..cbea2b7 100755 --- a/lib/multicore.js +++ b/lib/multicore.js @@ -9,13 +9,16 @@ export function getHalfCores() { return Math.round(totalCores / 2) || 1 } -export function runTask(cores, task) { +export async function runTask(cores, task) { + if (cores == 1) { + return await task() + } if (cluster.isMaster) { for (let index = 0; index < cores; index++) { cluster.fork() } } else { - task() + await task() } } From b61a82ce89c57cd3901285aace5e1b7ea5151e5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Fri, 18 Aug 2023 17:40:54 +0200 Subject: [PATCH 08/40] Run loadtest in multicore with --cores --- bin/loadtest.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/loadtest.js b/bin/loadtest.js index 264c643..63e7b9f 100755 --- a/bin/loadtest.js +++ b/bin/loadtest.js @@ -3,6 +3,7 @@ import {readFile} from 'fs/promises' import * as stdio from 'stdio' import {loadTest} from '../lib/loadtest.js' +import {runTask} from '../lib/multicore.js' const options = stdio.getopt({ @@ -32,8 +33,9 @@ const options = stdio.getopt({ key: {args: 1, description: 'The client key to use'}, cert: {args: 1, description: 'The client certificate to use'}, quiet: {description: 'Do not log any messages'}, + cores: {args: 1, description: 'Number of cores to use', default: 1}, agent: {description: 'Use a keep-alive http agent (deprecated)'}, - debug: {description: 'Show debug messages (deprecated)'} + debug: {description: 'Show debug messages (deprecated)'}, }); async function processAndRun(options) { @@ -51,7 +53,7 @@ async function processAndRun(options) { help(); } options.url = options.args[0]; - await startTest(options) + runTask(options.cores, async () => await startTest(options)) } async function startTest(options) { From 9d51eaace47e6a35b6806581a80d811c7ea4a903 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Fri, 18 Aug 2023 19:55:22 +0200 Subject: [PATCH 09/40] Do not use deprecated api --- lib/multicore.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/multicore.js b/lib/multicore.js index cbea2b7..f005fed 100755 --- a/lib/multicore.js +++ b/lib/multicore.js @@ -13,7 +13,7 @@ export async function runTask(cores, task) { if (cores == 1) { return await task() } - if (cluster.isMaster) { + if (cluster.isPrimary) { for (let index = 0; index < cores; index++) { cluster.fork() } From bf97b22ef218ab6d5dda6d45e609024227b4df8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Fri, 18 Aug 2023 19:56:12 +0200 Subject: [PATCH 10/40] Moved multicore to cluster --- bin/loadtest.js | 2 +- bin/testserver.js | 4 +--- lib/{multicore.js => cluster.js} | 0 3 files changed, 2 insertions(+), 4 deletions(-) rename lib/{multicore.js => cluster.js} (100%) diff --git a/bin/loadtest.js b/bin/loadtest.js index 63e7b9f..2956a70 100755 --- a/bin/loadtest.js +++ b/bin/loadtest.js @@ -3,7 +3,7 @@ import {readFile} from 'fs/promises' import * as stdio from 'stdio' import {loadTest} from '../lib/loadtest.js' -import {runTask} from '../lib/multicore.js' +import {runTask} from '../lib/cluster.js' const options = stdio.getopt({ diff --git a/bin/testserver.js b/bin/testserver.js index f4d08aa..a5da343 100755 --- a/bin/testserver.js +++ b/bin/testserver.js @@ -3,9 +3,7 @@ import * as stdio from 'stdio' import {startServer} from '../lib/testserver.js' import {loadConfig} from '../lib/config.js' -import * as cluster from 'cluster' -import {cpus} from 'os' -import {getHalfCores, runTask} from '../lib/multicore.js' +import {getHalfCores, runTask} from '../lib/cluster.js' const options = readOptions() start(options) diff --git a/lib/multicore.js b/lib/cluster.js similarity index 100% rename from lib/multicore.js rename to lib/cluster.js From 866b1ad7df17ba3090a350c5b4a8f3fab35cd791 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Sun, 20 Aug 2023 10:34:03 +0200 Subject: [PATCH 11/40] Make cluster work with scheduling policy none --- lib/cluster.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cluster.js b/lib/cluster.js index f005fed..cd211d7 100755 --- a/lib/cluster.js +++ b/lib/cluster.js @@ -1,7 +1,8 @@ -#!/usr/bin/env node +process.env.NODE_CLUSTER_SCHED_POLICY = 'none' -import * as cluster from 'cluster' import {cpus} from 'os' +// dynamic import as workaround: https://github.com/nodejs/node/issues/49240 +const cluster = await import('cluster') export function getHalfCores() { @@ -22,4 +23,3 @@ export async function runTask(cores, task) { } } - From 4d1e5b8acecbec0f406ba81a11406bc4d5494a8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Sun, 20 Aug 2023 11:48:46 +0200 Subject: [PATCH 12/40] Aggregate all results from all workers --- bin/loadtest.js | 5 ++++- lib/cluster.js | 28 +++++++++++++++++++++++----- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/bin/loadtest.js b/bin/loadtest.js index 2956a70..6ed3e07 100755 --- a/bin/loadtest.js +++ b/bin/loadtest.js @@ -53,13 +53,16 @@ async function processAndRun(options) { help(); } options.url = options.args[0]; - runTask(options.cores, async () => await startTest(options)) + const cores = parseInt(options.cores) || 1 + const results = await runTask(cores, async () => await startTest(options)) + console.log(results) } async function startTest(options) { try { const result = await loadTest(options) result.show() + return result } catch(error) { console.error(error.message) help() diff --git a/lib/cluster.js b/lib/cluster.js index cd211d7..2812d40 100755 --- a/lib/cluster.js +++ b/lib/cluster.js @@ -12,14 +12,32 @@ export function getHalfCores() { export async function runTask(cores, task) { if (cores == 1) { - return await task() + return [await task()] } if (cluster.isPrimary) { - for (let index = 0; index < cores; index++) { - cluster.fork() - } + return await runWorkers(cores) } else { - await task() + const result = await task() + console.log('Worker finished') + process.send(result) } } +function runWorkers(cores) { + return new Promise((resolve, reject) => { + const results = [] + for (let index = 0; index < cores; index++) { + const worker = cluster.fork() + worker.on('message', message => { + console.log('Received message', message) + results.push(message) + console.log(`Received ${results.length}, need ${cores}`) + if (results.length === cores) { + console.log('All messages received') + return resolve(results) + } + }) + } + }) +} + From db49e909f43878a76d3a532e35fbf866e79df7ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Sun, 20 Aug 2023 12:09:42 +0200 Subject: [PATCH 13/40] Function to combine results --- lib/result.js | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/result.js b/lib/result.js index 52d72ee..ade553e 100644 --- a/lib/result.js +++ b/lib/result.js @@ -56,6 +56,27 @@ export class Result { } } + combine(result) { + // configuration + this.url = this.url || result.url + this.maxRequests += result.maxRequests + this.maxSeconds = this.maxSeconds || result.maxSeconds + this.concurrency = this.concurrency || result.concurrency + this.agent = this.agent || result.agent + this.requestsPerSecond += result.requestsPerSecond + // result + this.totalRequests += result.totalRequests + this.totalErrors += result.totalErrors + this.elapsedSeconds = Math.max(this.elapsedSeconds, result.elapsedSeconds) + this.accumulatedMs += result.accumulatedMs + this.maxLatencyMs = Math.max(this.maxLatencyMs, result.maxLatencyMs) + this.minLatencyMs = Math.min(this.minLatencyMs, result.minLatencyMs) + this.combineMap(this.errorCodes, result.errorCodes) + this.combineMap(this.errorCodes, result.errorCodes) + this.histogramMs = result.histogramMs + this.computeDerived() + } + /** * Show result of a load test. */ @@ -75,7 +96,7 @@ export class Result { console.info(''); console.info('Completed requests: %s', this.totalRequests); console.info('Total errors: %s', this.totalErrors); - console.info('Total time: %s s', this.totalTimeSeconds); + console.info('Total time: %s s', this.elapsedSeconds); console.info('Requests per second: %s', this.rps); console.info('Mean latency: %s ms', this.meanLatencyMs); console.info(''); From 467efd2db9132e72b3d9fa0aa81351a0ee9f6aee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Sun, 20 Aug 2023 12:15:52 +0200 Subject: [PATCH 14/40] Reject result when there is an error in the cluster --- lib/cluster.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/cluster.js b/lib/cluster.js index 2812d40..a338208 100755 --- a/lib/cluster.js +++ b/lib/cluster.js @@ -37,6 +37,9 @@ function runWorkers(cores) { return resolve(results) } }) + worker.on('error', error => { + return reject(error) + }) } }) } From c40d9af48858babf2b6bf729300f8e79d7b16cf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Sun, 20 Aug 2023 12:16:19 +0200 Subject: [PATCH 15/40] New test to combine results --- test/result.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 test/result.js diff --git a/test/result.js b/test/result.js new file mode 100644 index 0000000..c3977da --- /dev/null +++ b/test/result.js @@ -0,0 +1,18 @@ +import testing from 'testing' +import {Result} from '../lib/result.js' + + +function testCombineResults(callback) { + const result = new Result() + result.combine(new Result()) + testing.assert(!result.url, callback) + testing.success(callback) +} + +export function test(callback) { + const tests = [ + testCombineResults, + ]; + testing.run(tests, callback); +} + From aa040f2898a7d7cb7b265f62147e4d6f6f06425c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Sun, 20 Aug 2023 12:16:31 +0200 Subject: [PATCH 16/40] New test for results --- test/all.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/all.js b/test/all.js index d25b919..e0ff851 100644 --- a/test/all.js +++ b/test/all.js @@ -14,6 +14,7 @@ import {test as testBodyGenerator} from './body-generator.js' import {test as testLoadtest} from './loadtest.js' import {test as testWebsocket} from './websocket.js' import {test as integrationTest} from './integration.js' +import {test as testResult} from './result.js' /** @@ -23,7 +24,7 @@ function test() { const tests = [ testHrtimer, testHeaders, testLatency, testHttpClient, testServer, integrationTest, testLoadtest, testWebsocket, - testRequestGenerator, testBodyGenerator, + testRequestGenerator, testBodyGenerator, testResult, ]; testing.run(tests, 4200); } From b86ed9eb776ce185f516f49ccbd89dcd29868255 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Sun, 20 Aug 2023 12:16:42 +0200 Subject: [PATCH 17/40] Combine results in map --- lib/latency.js | 3 ++- lib/result.js | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/lib/latency.js b/lib/latency.js index 64745ab..1b3e712 100644 --- a/lib/latency.js +++ b/lib/latency.js @@ -174,7 +174,8 @@ export class Latency { * Get final result. */ getResult() { - const result = new Result(this.options, this) + const result = new Result() + result.compute(this.options, this) return result } diff --git a/lib/result.js b/lib/result.js index ade553e..393f9d3 100644 --- a/lib/result.js +++ b/lib/result.js @@ -4,7 +4,10 @@ * Result of a load test. */ export class Result { - constructor(options, latency) { + constructor() { + } + + compute(options, latency) { // configuration this.url = options.url this.maxRequests = options.maxRequests @@ -77,6 +80,17 @@ export class Result { this.computeDerived() } + combineMap(originalMap, addedMap) { + for (const key in {...originalMap, ...addedMap}) { + if (!originalMap[key]) { + originalMap[key] = 0 + } + if (addedMap[key]) { + originalMap[key] += addedMap[key] + } + } + } + /** * Show result of a load test. */ From 390dba6dfcbe55cfce79c156de3821aa134f68dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Sun, 20 Aug 2023 12:26:31 +0200 Subject: [PATCH 18/40] Add test for empty and complex results --- test/result.js | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/test/result.js b/test/result.js index c3977da..1dedaf1 100644 --- a/test/result.js +++ b/test/result.js @@ -2,16 +2,44 @@ import testing from 'testing' import {Result} from '../lib/result.js' -function testCombineResults(callback) { +function testCombineEmptyResults(callback) { const result = new Result() result.combine(new Result()) testing.assert(!result.url, callback) testing.success(callback) } +function testCombineResults(callback) { + const combined = new Result() + const url = 'https://pinchito.es/' + for (let index = 0; index < 3; index++) { + const result = { + url, + cores: 7, + maxRequests: 1000, + concurrency: 10, + agent: 'none', + requestsPerSecond: 100, + totalRequests: 330, + totalErrors: 10, + totalTimeSeconds: 5, + accumulatedMs: 5000, + maxLatencyMs: 350 + index, + minLatencyMs: index + 2, + errorCodes: {200: 100, 100: 200}, + histogramMs: {2: 1, 3: 4, 100: 300}, + } + combined.combine(result) + } + testing.assertEquals(combined.url, url, callback) + testing.assertEquals(combined.cores, 3, callback) + testing.assertEquals(combined.totalErrors, 30, callback) + testing.success(callback) +} + export function test(callback) { const tests = [ - testCombineResults, + testCombineEmptyResults, testCombineResults, ]; testing.run(tests, callback); } From 320a1521a7316af6a3a2b1b2d9ef13d2a55579be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Sun, 20 Aug 2023 12:26:49 +0200 Subject: [PATCH 19/40] Show how many cores the test run on --- lib/result.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/result.js b/lib/result.js index 393f9d3..a5043fe 100644 --- a/lib/result.js +++ b/lib/result.js @@ -10,6 +10,7 @@ export class Result { compute(options, latency) { // configuration this.url = options.url + this.cores = options.cores this.maxRequests = options.maxRequests this.maxSeconds = options.maxSeconds this.concurrency = options.concurrency @@ -62,6 +63,7 @@ export class Result { combine(result) { // configuration this.url = this.url || result.url + this.cores += 1 this.maxRequests += result.maxRequests this.maxSeconds = this.maxSeconds || result.maxSeconds this.concurrency = this.concurrency || result.concurrency @@ -103,6 +105,7 @@ export class Result { console.info('Max time (s): %s', this.maxSeconds); } console.info('Concurrency level: %s', this.concurrency); + console.info('Running on cores: %s', this.cores); console.info('Agent: %s', this.agent); if (this.requestsPerSecond) { console.info('Requests per second: %s', this.requestsPerSecond); From 4e79d0790e5fe19e5835131c3558a50540084ff1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Sun, 20 Aug 2023 12:26:57 +0200 Subject: [PATCH 20/40] Combine histogram correctly --- lib/result.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/result.js b/lib/result.js index a5043fe..ee40557 100644 --- a/lib/result.js +++ b/lib/result.js @@ -77,8 +77,7 @@ export class Result { this.maxLatencyMs = Math.max(this.maxLatencyMs, result.maxLatencyMs) this.minLatencyMs = Math.min(this.minLatencyMs, result.minLatencyMs) this.combineMap(this.errorCodes, result.errorCodes) - this.combineMap(this.errorCodes, result.errorCodes) - this.histogramMs = result.histogramMs + this.combineMap(this.histogramMs, result.histogramMs) this.computeDerived() } From de626d89a787cbc81264b2eaadb2c89b8b52a3d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Sun, 20 Aug 2023 12:34:26 +0200 Subject: [PATCH 21/40] Reset all values before combining --- lib/result.js | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/result.js b/lib/result.js index ee40557..7c0892c 100644 --- a/lib/result.js +++ b/lib/result.js @@ -5,6 +5,22 @@ */ export class Result { constructor() { + this.url = null + this.cores = 0 + this.maxRequests = 0 + this.maxSeconds = 0 + this.concurrency = 0 + this.agent = null + this.requestsPerSecond = 0 + this.elapsedSeconds = 0 + this.totalRequests = 0 + this.totalErrors = 0 + this.totalTimeSeconds = 0 + this.accumulatedMs = 0 + this.maxLatencyMs = 0 + this.minLatencyMs = Number.MAX_SAFE_INTEGER + this.errorCodes = {} + this.histogramMs = {} } compute(options, latency) { @@ -68,11 +84,12 @@ export class Result { this.maxSeconds = this.maxSeconds || result.maxSeconds this.concurrency = this.concurrency || result.concurrency this.agent = this.agent || result.agent - this.requestsPerSecond += result.requestsPerSecond + this.requestsPerSecond += result.requestsPerSecond || 0 // result this.totalRequests += result.totalRequests this.totalErrors += result.totalErrors this.elapsedSeconds = Math.max(this.elapsedSeconds, result.elapsedSeconds) + this.totalTimeSeconds = this.elapsedSeconds this.accumulatedMs += result.accumulatedMs this.maxLatencyMs = Math.max(this.maxLatencyMs, result.maxLatencyMs) this.minLatencyMs = Math.min(this.minLatencyMs, result.minLatencyMs) From fb3aa968396621e003f7d4f610a7a8e59027e162 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Sun, 20 Aug 2023 12:34:40 +0200 Subject: [PATCH 22/40] Check elapsed seconds --- test/result.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/result.js b/test/result.js index 1dedaf1..c4710b6 100644 --- a/test/result.js +++ b/test/result.js @@ -22,10 +22,10 @@ function testCombineResults(callback) { requestsPerSecond: 100, totalRequests: 330, totalErrors: 10, - totalTimeSeconds: 5, + elapsedSeconds: 5 + index, accumulatedMs: 5000, maxLatencyMs: 350 + index, - minLatencyMs: index + 2, + minLatencyMs: 2 + index, errorCodes: {200: 100, 100: 200}, histogramMs: {2: 1, 3: 4, 100: 300}, } @@ -34,6 +34,7 @@ function testCombineResults(callback) { testing.assertEquals(combined.url, url, callback) testing.assertEquals(combined.cores, 3, callback) testing.assertEquals(combined.totalErrors, 30, callback) + testing.assertEquals(combined.elapsedSeconds, 7, callback) testing.success(callback) } From 6d9a1c6c6cbd80c700c939b59f591c9f7059c750 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Sun, 20 Aug 2023 13:05:18 +0200 Subject: [PATCH 23/40] Show results from workers --- bin/loadtest.js | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/bin/loadtest.js b/bin/loadtest.js index 6ed3e07..11ac5d8 100755 --- a/bin/loadtest.js +++ b/bin/loadtest.js @@ -4,6 +4,7 @@ import {readFile} from 'fs/promises' import * as stdio from 'stdio' import {loadTest} from '../lib/loadtest.js' import {runTask} from '../lib/cluster.js' +import {Result} from '../lib/result.js' const options = stdio.getopt({ @@ -55,20 +56,37 @@ async function processAndRun(options) { options.url = options.args[0]; const cores = parseInt(options.cores) || 1 const results = await runTask(cores, async () => await startTest(options)) - console.log(results) + if (!results) { + process.exit(0) + return + } + console.trace('***') + console.log(results.length) + showResults(results) } async function startTest(options) { try { - const result = await loadTest(options) - result.show() - return result + return await loadTest(options) } catch(error) { console.error(error.message) help() } } +function showResults(results) { + console.log(results.length) + if (results.length == 1) { + results[0].show() + return + } + const combined = new Result() + for (const result of results) { + combined.combine(result) + } + combined.show() +} + await processAndRun(options) function help() { From 1ff3776a7f25fc542ca30e85bfea87ef6e207300 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Sun, 20 Aug 2023 13:35:31 +0200 Subject: [PATCH 24/40] Share values amongst cores --- bin/loadtest.js | 6 +++--- lib/result.js | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bin/loadtest.js b/bin/loadtest.js index 11ac5d8..b9c8a9f 100755 --- a/bin/loadtest.js +++ b/bin/loadtest.js @@ -54,14 +54,15 @@ async function processAndRun(options) { help(); } options.url = options.args[0]; + // share values amongst cores const cores = parseInt(options.cores) || 1 + options.maxRequests = parseInt(options.maxRequests) / cores + options.requestsPerSecond = parseInt(options.requestsPerSecond) / cores const results = await runTask(cores, async () => await startTest(options)) if (!results) { process.exit(0) return } - console.trace('***') - console.log(results.length) showResults(results) } @@ -75,7 +76,6 @@ async function startTest(options) { } function showResults(results) { - console.log(results.length) if (results.length == 1) { results[0].show() return diff --git a/lib/result.js b/lib/result.js index 7c0892c..08467e2 100644 --- a/lib/result.js +++ b/lib/result.js @@ -27,11 +27,11 @@ export class Result { // configuration this.url = options.url this.cores = options.cores - this.maxRequests = options.maxRequests - this.maxSeconds = options.maxSeconds - this.concurrency = options.concurrency + this.maxRequests = parseInt(options.maxRequests) + this.maxSeconds = parseInt(options.maxSeconds) + this.concurrency = parseInt(options.concurrency) this.agent = options.agentKeepAlive ? 'keepalive' : 'none'; - this.requestsPerSecond = options.requestsPerSecond + this.requestsPerSecond = parseInt(options.requestsPerSecond) // result this.elapsedSeconds = latency.getElapsed(latency.initialTime) / 1000 this.totalRequests = latency.totalRequests From e9bba9315161b6799ca4d1194544e5e290d1e9b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Sun, 20 Aug 2023 13:35:37 +0200 Subject: [PATCH 25/40] Reorder --- lib/testserver.js | 48 +++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/lib/testserver.js b/lib/testserver.js index 4fc7aaa..ce4b0fa 100644 --- a/lib/testserver.js +++ b/lib/testserver.js @@ -8,6 +8,30 @@ const PORT = 7357; const LOG_HEADERS_INTERVAL_MS = 5000; +/** + * Start a test server. Parameters: + * - `options`, can contain: + * - port: the port to use, default 7357. + * - delay: wait the given milliseconds before answering. + * - quiet: do not log any messages. + * - percent: give an error (default 500) on some % of requests. + * - error: set an HTTP error code, default is 500. + * - `callback`: optional callback, called after the server has started. + * If not present will return a promise. + */ +export function startServer(options, callback) { + const server = new TestServer(options); + if (callback) { + return server.start(callback) + } + return new Promise((resolve, reject) => { + server.start((error, result) => { + if (error) return reject(error) + return resolve(result) + }) + }) +} + /** * A test server, with the given options (see below on startServer()). */ @@ -165,27 +189,3 @@ class TestServer { } } -/** - * Start a test server. Parameters: - * - `options`, can contain: - * - port: the port to use, default 7357. - * - delay: wait the given milliseconds before answering. - * - quiet: do not log any messages. - * - percent: give an error (default 500) on some % of requests. - * - error: set an HTTP error code, default is 500. - * - `callback`: optional callback, called after the server has started. - * If not present will return a promise. - */ -export function startServer(options, callback) { - const server = new TestServer(options); - if (callback) { - return server.start(callback) - } - return new Promise((resolve, reject) => { - server.start((error, result) => { - if (error) return reject(error) - return resolve(result) - }) - }) -} - From 5a10e1e907f5b2c68a19d3fb867c8d07e87e6fd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Sun, 20 Aug 2023 13:35:44 +0200 Subject: [PATCH 26/40] Wait for server to start --- bin/testserver.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/testserver.js b/bin/testserver.js index a5da343..09040c4 100755 --- a/bin/testserver.js +++ b/bin/testserver.js @@ -47,7 +47,7 @@ function readOptions() { } function start(options) { - runTask(options.cores, () => startServer(options)) + runTask(options.cores, async () => await startServer(options)) } From ccedac8d2a3936871ba66fc2711fd904018dacf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Sun, 20 Aug 2023 13:43:07 +0200 Subject: [PATCH 27/40] Remove traces --- lib/cluster.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/cluster.js b/lib/cluster.js index a338208..63f0515 100755 --- a/lib/cluster.js +++ b/lib/cluster.js @@ -18,7 +18,6 @@ export async function runTask(cores, task) { return await runWorkers(cores) } else { const result = await task() - console.log('Worker finished') process.send(result) } } @@ -29,11 +28,8 @@ function runWorkers(cores) { for (let index = 0; index < cores; index++) { const worker = cluster.fork() worker.on('message', message => { - console.log('Received message', message) results.push(message) - console.log(`Received ${results.length}, need ${cores}`) if (results.length === cores) { - console.log('All messages received') return resolve(results) } }) From f29b1e63ba4bb462efcbc4b8cc0519b7d0e8cc47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Sun, 20 Aug 2023 15:15:02 +0200 Subject: [PATCH 28/40] Divide max requests and rps by cores only if present --- bin/loadtest.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/loadtest.js b/bin/loadtest.js index b9c8a9f..e22de62 100755 --- a/bin/loadtest.js +++ b/bin/loadtest.js @@ -56,8 +56,8 @@ async function processAndRun(options) { options.url = options.args[0]; // share values amongst cores const cores = parseInt(options.cores) || 1 - options.maxRequests = parseInt(options.maxRequests) / cores - options.requestsPerSecond = parseInt(options.requestsPerSecond) / cores + options.maxRequests = options.maxRequests ? parseInt(options.maxRequests) / cores : null + options.rps = options.rps ? parseInt(options.rps) / cores : null const results = await runTask(cores, async () => await startTest(options)) if (!results) { process.exit(0) From 43b4c92062f83c097a7946fae8a828363e71b863 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Sun, 20 Aug 2023 19:02:19 +0200 Subject: [PATCH 29/40] Share requests and rps properly among cores --- bin/loadtest.js | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/bin/loadtest.js b/bin/loadtest.js index e22de62..6ac6ea9 100755 --- a/bin/loadtest.js +++ b/bin/loadtest.js @@ -54,11 +54,8 @@ async function processAndRun(options) { help(); } options.url = options.args[0]; - // share values amongst cores - const cores = parseInt(options.cores) || 1 - options.maxRequests = options.maxRequests ? parseInt(options.maxRequests) / cores : null - options.rps = options.rps ? parseInt(options.rps) / cores : null - const results = await runTask(cores, async () => await startTest(options)) + computeCores(options) + const results = await runTask(options.cores, async () => await startTest(options)) if (!results) { process.exit(0) return @@ -66,6 +63,18 @@ async function processAndRun(options) { showResults(results) } +function computeCores(options) { + options.cores = parseInt(options.cores) || 1 + if (options.maxRequests) { + const maxRequests = parseInt(options.maxRequests) + options.maxRequests = Math.round(maxRequests / options.cores) + } + if (options.rps) { + const rps = parseInt(options.rps) + options.rps = Math.round(rps / options.cores) + } +} + async function startTest(options) { try { return await loadTest(options) From e06415f1535b7fc2b4edffb64db7fd463157bb49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Sun, 20 Aug 2023 19:54:07 +0200 Subject: [PATCH 30/40] Share rps and max requests properly between cores --- bin/loadtest.js | 56 +++++++++++++++++++++++++++++-------------------- lib/cluster.js | 2 +- 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/bin/loadtest.js b/bin/loadtest.js index 6ac6ea9..1821a54 100755 --- a/bin/loadtest.js +++ b/bin/loadtest.js @@ -54,8 +54,8 @@ async function processAndRun(options) { help(); } options.url = options.args[0]; - computeCores(options) - const results = await runTask(options.cores, async () => await startTest(options)) + options.cores = parseInt(options.cores) || 1 + const results = await runTask(options.cores, async workerId => await startTest(options, workerId)) if (!results) { process.exit(0) return @@ -63,27 +63,6 @@ async function processAndRun(options) { showResults(results) } -function computeCores(options) { - options.cores = parseInt(options.cores) || 1 - if (options.maxRequests) { - const maxRequests = parseInt(options.maxRequests) - options.maxRequests = Math.round(maxRequests / options.cores) - } - if (options.rps) { - const rps = parseInt(options.rps) - options.rps = Math.round(rps / options.cores) - } -} - -async function startTest(options) { - try { - return await loadTest(options) - } catch(error) { - console.error(error.message) - help() - } -} - function showResults(results) { if (results.length == 1) { results[0].show() @@ -96,6 +75,37 @@ function showResults(results) { combined.show() } +async function startTest(options, workerId) { + if (!workerId) { + // standalone; controlled errors + try { + return await loadTest(options) + } catch(error) { + console.error(error.message) + return help() + } + } + shareWorker(options, workerId) + return await loadTest(options) +} + +function shareWorker(options, workerId) { + options.maxRequests = shareOption(options.maxRequests, workerId, options.cores) + options.rps = shareOption(options.rps, workerId, options.cores) +} + +function shareOption(option, workerId, cores) { + if (!option) return null + const total = parseInt(option) + const shared = Math.round(total / cores) + if (workerId == cores) { + // last worker gets remainder + return total - shared * (cores - 1) + } else { + return shared + } +} + await processAndRun(options) function help() { diff --git a/lib/cluster.js b/lib/cluster.js index 63f0515..148e956 100755 --- a/lib/cluster.js +++ b/lib/cluster.js @@ -17,7 +17,7 @@ export async function runTask(cores, task) { if (cluster.isPrimary) { return await runWorkers(cores) } else { - const result = await task() + const result = await task(cluster.worker.id) process.send(result) } } From 984602070d2dad41a34f31a002be4c7db5723335 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Sun, 20 Aug 2023 19:56:57 +0200 Subject: [PATCH 31/40] Show target and effective rps --- lib/result.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/result.js b/lib/result.js index 08467e2..a6ef0b2 100644 --- a/lib/result.js +++ b/lib/result.js @@ -36,7 +36,7 @@ export class Result { this.elapsedSeconds = latency.getElapsed(latency.initialTime) / 1000 this.totalRequests = latency.totalRequests this.totalErrors = latency.totalErrors - this.totalTimeSeconds = this.elapsedSeconds + this.totalTimeSeconds = this.elapsedSeconds // backwards compatibility this.accumulatedMs = latency.totalTime this.maxLatencyMs = latency.maxLatencyMs this.minLatencyMs = latency.minLatencyMs @@ -48,7 +48,8 @@ export class Result { computeDerived() { const meanTime = this.accumulatedMs / this.totalRequests this.meanLatencyMs = Math.round(meanTime * 10) / 10 - this.rps = Math.round(this.totalRequests / this.elapsedSeconds) + this.effectiveRps = Math.round(this.totalRequests / this.elapsedSeconds) + this.rps = this.effectiveRps // backwards compatibility this.computePercentiles() } @@ -124,13 +125,13 @@ export class Result { console.info('Running on cores: %s', this.cores); console.info('Agent: %s', this.agent); if (this.requestsPerSecond) { - console.info('Requests per second: %s', this.requestsPerSecond); + console.info('Target rps: %s', this.requestsPerSecond); } console.info(''); console.info('Completed requests: %s', this.totalRequests); console.info('Total errors: %s', this.totalErrors); console.info('Total time: %s s', this.elapsedSeconds); - console.info('Requests per second: %s', this.rps); + console.info('Effective rps: %s', this.effectiveRps); console.info('Mean latency: %s ms', this.meanLatencyMs); console.info(''); console.info('Percentage of the requests served within a certain time'); From 1853c6d3a92eec9006d99c055f7911fe0181d923 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Sun, 20 Aug 2023 22:34:34 +0200 Subject: [PATCH 32/40] Show effective rps last --- lib/result.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/result.js b/lib/result.js index a6ef0b2..a9665a0 100644 --- a/lib/result.js +++ b/lib/result.js @@ -131,8 +131,8 @@ export class Result { console.info('Completed requests: %s', this.totalRequests); console.info('Total errors: %s', this.totalErrors); console.info('Total time: %s s', this.elapsedSeconds); - console.info('Effective rps: %s', this.effectiveRps); console.info('Mean latency: %s ms', this.meanLatencyMs); + console.info('Effective rps: %s', this.effectiveRps); console.info(''); console.info('Percentage of the requests served within a certain time'); From ed23b94e3121e40522d0559debb405a14442d6aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Sun, 20 Aug 2023 22:51:08 +0200 Subject: [PATCH 33/40] Rename --- lib/latency.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/latency.js b/lib/latency.js index 1b3e712..bf78a8c 100644 --- a/lib/latency.js +++ b/lib/latency.js @@ -18,7 +18,7 @@ export class Latency { this.partialTime = 0; this.partialErrors = 0; this.lastShown = this.getTime(); - this.initialTime = this.getTime(); + this.startTime = this.getTime(); this.totalRequests = 0; this.totalTime = 0; this.totalErrors = 0; @@ -153,7 +153,7 @@ export class Latency { if (this.options.maxRequests && this.totalRequests >= this.options.maxRequests) { return true; } - const elapsedSeconds = this.getElapsed(this.initialTime) / 1000; + const elapsedSeconds = this.getElapsed(this.startTime) / 1000; if (this.options.maxSeconds && elapsedSeconds >= this.options.maxSeconds) { return true; } From 3f07b2905dce95ef330a483889b50532014349ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Mon, 21 Aug 2023 00:17:42 +0200 Subject: [PATCH 34/40] Store start and end times in ns and ms --- lib/latency.js | 32 +++++++++++++++++--------------- lib/loadtest.js | 2 +- lib/result.js | 6 +++++- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/lib/latency.js b/lib/latency.js index bf78a8c..d51b877 100644 --- a/lib/latency.js +++ b/lib/latency.js @@ -17,8 +17,8 @@ export class Latency { this.partialRequests = 0; this.partialTime = 0; this.partialErrors = 0; - this.lastShown = this.getTime(); - this.startTime = this.getTime(); + this.lastShownNs = this.getTimeNs(); + this.startTimeNs = this.getTimeNs(); this.totalRequests = 0; this.totalTime = 0; this.totalErrors = 0; @@ -45,7 +45,7 @@ export class Latency { */ start(requestId) { requestId = requestId || createId(); - this.requests[requestId] = this.getTime(); + this.requests[requestId] = this.getTimeNs(); this.requestIdToIndex[requestId] = this.requestIndex++; return requestId; } @@ -61,7 +61,7 @@ export class Latency { if (!this.running) { return -1; } - const elapsed = this.getElapsed(this.requests[requestId]); + const elapsed = this.getElapsedMs(this.requests[requestId]); this.add(elapsed, errorCode); delete this.requests[requestId]; return elapsed; @@ -105,7 +105,7 @@ export class Latency { * Show latency for partial requests. */ showPartial() { - const elapsedSeconds = this.getElapsed(this.lastShown) / 1000; + const elapsedSeconds = this.getElapsedMs(this.lastShownNs) / 1000; const meanTime = this.partialTime / this.partialRequests || 0.0; const result = { meanLatencyMs: Math.round(meanTime * 10) / 10, @@ -125,25 +125,26 @@ export class Latency { this.partialTime = 0; this.partialRequests = 0; this.partialErrors = 0; - this.lastShown = this.getTime(); + this.lastShownNs = this.getTimeNs(); } /** - * Returns the current high-resolution real time in a [seconds, nanoseconds] tuple Array + * Returns the current high-resolution real time in nanoseconds as a big int. * @return {*} */ - getTime() { - return process.hrtime(); + getTimeNs() { + return process.hrtime.bigint(); } /** - * calculates the elapsed time between the assigned startTime and now - * @param startTime + * Calculates the elapsed time between the assigned start time and now in ms. + * @param startTimeNs time in nanoseconds (bigint) * @return {Number} the elapsed time in milliseconds */ - getElapsed(startTime) { - const elapsed = process.hrtime(startTime); - return elapsed[0] * 1000 + elapsed[1] / 1000000; + getElapsedMs(startTimeNs) { + const endTimeNs = this.getTimeNs() + const elapsedNs = endTimeNs - startTimeNs + return Number(elapsedNs / 1000000n) } /** @@ -153,7 +154,7 @@ export class Latency { if (this.options.maxRequests && this.totalRequests >= this.options.maxRequests) { return true; } - const elapsedSeconds = this.getElapsed(this.startTime) / 1000; + const elapsedSeconds = this.getElapsedMs(this.startTimeNs) / 1000; if (this.options.maxSeconds && elapsedSeconds >= this.options.maxSeconds) { return true; } @@ -165,6 +166,7 @@ export class Latency { */ finish() { this.running = false; + this.endTimeNs = this.getTimeNs() if (this.callback) { return this.callback(null, this.getResult()); } diff --git a/lib/loadtest.js b/lib/loadtest.js index 04f501b..68510bd 100644 --- a/lib/loadtest.js +++ b/lib/loadtest.js @@ -160,7 +160,7 @@ class Operation { */ stop() { this.running = false; - this.latency.running = false; + this.latency.finish() if (this.showTimer) { this.showTimer.stop(); } diff --git a/lib/result.js b/lib/result.js index a9665a0..3df3c8f 100644 --- a/lib/result.js +++ b/lib/result.js @@ -12,6 +12,8 @@ export class Result { this.concurrency = 0 this.agent = null this.requestsPerSecond = 0 + this.startTimeMs = Number.MAX_SAFE_INTEGER + this.endTimeMs = 0 this.elapsedSeconds = 0 this.totalRequests = 0 this.totalErrors = 0 @@ -33,7 +35,9 @@ export class Result { this.agent = options.agentKeepAlive ? 'keepalive' : 'none'; this.requestsPerSecond = parseInt(options.requestsPerSecond) // result - this.elapsedSeconds = latency.getElapsed(latency.initialTime) / 1000 + this.startTimeMs = Number(latency.startTimeNs / 1000000n) + this.endTimeMs = Number(latency.endTimeNs / 1000000n) + this.elapsedSeconds = (this.endTimeMs - this.startTimeMs) / 1000 this.totalRequests = latency.totalRequests this.totalErrors = latency.totalErrors this.totalTimeSeconds = this.elapsedSeconds // backwards compatibility From 535a8065336340d2de8d8f6e0f4400bbe7dd3073 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Mon, 21 Aug 2023 00:21:27 +0200 Subject: [PATCH 35/40] Compute elapsed seconds as derivative of start and end times --- lib/result.js | 8 ++++---- test/result.js | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/result.js b/lib/result.js index 3df3c8f..7b16d7f 100644 --- a/lib/result.js +++ b/lib/result.js @@ -37,10 +37,8 @@ export class Result { // result this.startTimeMs = Number(latency.startTimeNs / 1000000n) this.endTimeMs = Number(latency.endTimeNs / 1000000n) - this.elapsedSeconds = (this.endTimeMs - this.startTimeMs) / 1000 this.totalRequests = latency.totalRequests this.totalErrors = latency.totalErrors - this.totalTimeSeconds = this.elapsedSeconds // backwards compatibility this.accumulatedMs = latency.totalTime this.maxLatencyMs = latency.maxLatencyMs this.minLatencyMs = latency.minLatencyMs @@ -50,6 +48,8 @@ export class Result { } computeDerived() { + this.elapsedSeconds = (this.endTimeMs - this.startTimeMs) / 1000 + this.totalTimeSeconds = this.elapsedSeconds // backwards compatibility const meanTime = this.accumulatedMs / this.totalRequests this.meanLatencyMs = Math.round(meanTime * 10) / 10 this.effectiveRps = Math.round(this.totalRequests / this.elapsedSeconds) @@ -91,10 +91,10 @@ export class Result { this.agent = this.agent || result.agent this.requestsPerSecond += result.requestsPerSecond || 0 // result + this.startTimeMs = Math.min(this.startTimeMs, result.startTimeMs) + this.endTimeMs = Math.max(this.endTimeMs, result.endTimeMs) this.totalRequests += result.totalRequests this.totalErrors += result.totalErrors - this.elapsedSeconds = Math.max(this.elapsedSeconds, result.elapsedSeconds) - this.totalTimeSeconds = this.elapsedSeconds this.accumulatedMs += result.accumulatedMs this.maxLatencyMs = Math.max(this.maxLatencyMs, result.maxLatencyMs) this.minLatencyMs = Math.min(this.minLatencyMs, result.minLatencyMs) diff --git a/test/result.js b/test/result.js index c4710b6..01b04fd 100644 --- a/test/result.js +++ b/test/result.js @@ -22,7 +22,8 @@ function testCombineResults(callback) { requestsPerSecond: 100, totalRequests: 330, totalErrors: 10, - elapsedSeconds: 5 + index, + startTimeMs: 1000 + index * 1000, + endTimeMs: 1000 + index * 2000, accumulatedMs: 5000, maxLatencyMs: 350 + index, minLatencyMs: 2 + index, @@ -34,7 +35,7 @@ function testCombineResults(callback) { testing.assertEquals(combined.url, url, callback) testing.assertEquals(combined.cores, 3, callback) testing.assertEquals(combined.totalErrors, 30, callback) - testing.assertEquals(combined.elapsedSeconds, 7, callback) + testing.assertEquals(combined.elapsedSeconds, 4, callback) testing.success(callback) } From 53149d66cdf410b03b2eb21a1357270cfc9eed90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Mon, 21 Aug 2023 09:58:34 +0200 Subject: [PATCH 36/40] Improve docs for result --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6f583ec..a58ebe9 100644 --- a/README.md +++ b/README.md @@ -528,7 +528,7 @@ and will not call the callback. The latency result returned at the end of the load test contains a full set of data, including: mean latency, number of errors and percentiles. -An example follows: +A simplified example follows: ```javascript { @@ -545,8 +545,8 @@ An example follows: '95': 11, '99': 15 }, - rps: 2824, - totalTimeSeconds: 0.354108, + effectiveRps: 2824, + elapsedSeconds: 0.354108, meanLatencyMs: 7.72, maxLatencyMs: 20, totalErrors: 3, From 22480ea25a1808b48dbaea0562ef3a4407cc710d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Mon, 21 Aug 2023 10:06:57 +0200 Subject: [PATCH 37/40] Show cores only if specified --- lib/result.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/result.js b/lib/result.js index 7b16d7f..5313c5f 100644 --- a/lib/result.js +++ b/lib/result.js @@ -126,7 +126,9 @@ export class Result { console.info('Max time (s): %s', this.maxSeconds); } console.info('Concurrency level: %s', this.concurrency); - console.info('Running on cores: %s', this.cores); + if (this.cores) { + console.info('Running on cores: %s', this.cores); + } console.info('Agent: %s', this.agent); if (this.requestsPerSecond) { console.info('Target rps: %s', this.requestsPerSecond); From b1d49742ef9cc1bf53fba0c0818f63347b666812 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Mon, 21 Aug 2023 10:15:52 +0200 Subject: [PATCH 38/40] Document --cores --- README.md | 55 ++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index a58ebe9..88a39d7 100644 --- a/README.md +++ b/README.md @@ -90,9 +90,13 @@ so that you can abort deployment e.g. if 99% of the requests don't finish in 10 ### Usage Don'ts `loadtest` saturates a single CPU pretty quickly. -Do not use `loadtest` if the Node.js process is above 100% usage in `top`, which happens approx. when your load is above 1000~4000 rps. +Do not use `loadtest` in this mode +if the Node.js process is above 100% usage in `top`, which happens approx. when your load is above 1000~4000 rps. (You can measure the practical limits of `loadtest` on your specific test machines by running it against a simple -Apache or nginx process and seeing when it reaches 100% CPU.) +[test server](#test-server) +and seeing when it reaches 100% CPU.) +In this case try using in multi-process mode using the `--cores` parameter, +see below. There are better tools for that use case: @@ -260,8 +264,9 @@ The following parameters are _not_ compatible with Apache ab. #### `--rps requestsPerSecond` Controls the number of requests per second that are sent. -Can be fractional, e.g. `--rps 0.5` sends one request every two seconds. -Not used by default: each request is sent as soon as the previous one is responded. +Cannot be fractional, e.g. `--rps 0.5`. +In this mode each request is not sent as soon as the previous one is responded, +but periodically even if previous requests have not been responded yet. Note: Concurrency doesn't affect the final number of requests per second, since rps will be shared by all the clients. E.g.: @@ -276,6 +281,16 @@ to send all of the rps, adjust it with `-c` if needed. Note: --rps is not supported for websockets. +#### `--cores number` + +Start `loadtest` in multi-process mode on a number of cores simultaneously. +Useful when a single CPU is saturated. +Forks the requested number of processes using the +[Node.js cluster module](https://nodejs.org/api/cluster.html). + +In this mode the total number of requests and the rps rate are shared among all processes. +The result returned is the aggregation of results from all cores. + #### `--timeout milliseconds` Timeout for each generated request in milliseconds. @@ -337,11 +352,11 @@ Sets the certificate for the http client to use. Must be used with `--key`. Sets the key for the http client to use. Must be used with `--cert`. -### Server +### Test Server loadtest bundles a test server. To run it: - $ testserver-loadtest [--delay ms] [error 5xx] [percent yy] [port] + $ testserver-loadtest [options] [port] This command will show the number of requests received per second, the latency in answering requests and the headers for selected requests. @@ -354,6 +369,27 @@ The optional delay instructs the server to wait for the given number of millisec before answering each request, to simulate a busy server. You can also simulate errors on a given percent of requests. +The following optional parameters are available. + +#### `--delay ms` + +Wait the specified number of milliseconds before answering each request. + +#### `--error 5xx` + +Return the given error for every request. + +#### `--percent yy` + +Return an error (default 500) only for the specified % of requests. + +#### `--cores number` + +Number of cores to use. If not 1, will start in multi-process mode. + +Note: since version v6.3.0 the test server uses half the available cores by default; +use `--cores 1` to use in single-process mode. + ### Complete Example Let us now see how to measure the performance of the test server. @@ -364,8 +400,9 @@ First we install `loadtest` globally: Now we start the test server: - $ testserver-loadtest - Listening on port 7357 + $ testserver-loadtest --cores 2 + Listening on http://localhost:7357/ + Listening on http://localhost:7357/ On a different console window we run a load test against it for 20 seconds with concurrency 10 (only relevant results are shown): @@ -458,7 +495,7 @@ The result (with the same test server) is impressive: 99% 10 ms 100% 25 ms (longest request) -Now you're talking! The steady rate also goes up to 2 krps: +Now we're talking! The steady rate also goes up to 2 krps: $ loadtest http://localhost:7357/ -t 20 -c 10 --keepalive --rps 2000 ... From 772f0b33ff9a80988b2cf23cf96df541521ba2c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Mon, 21 Aug 2023 10:15:56 +0200 Subject: [PATCH 39/40] v6.3.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4047b6b..220d239 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "loadtest", - "version": "6.2.2", + "version": "6.3.0", "type": "module", "description": "Run load tests for your web application. Mostly ab-compatible interface, with an option to force requests per second. Includes an API for automated load testing.", "homepage": "https://github.com/alexfernandez/loadtest", From 1de8e4e7626556fba429ae506c979413fa5a3dc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Fern=C3=A1ndez?= Date: Mon, 21 Aug 2023 10:24:49 +0200 Subject: [PATCH 40/40] Clarify --cores in the API --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 88a39d7..8ea2766 100644 --- a/README.md +++ b/README.md @@ -291,6 +291,9 @@ Forks the requested number of processes using the In this mode the total number of requests and the rps rate are shared among all processes. The result returned is the aggregation of results from all cores. +Note: this option is not available in the API, +where it runs just in the provided process. + #### `--timeout milliseconds` Timeout for each generated request in milliseconds.