feat: Added errorThreshold option.

ShogunPanda · Jan 29, 2020 · b3b38b2 · b3b38b2
1 parent 07e8f58
commit b3b38b2
Show file tree

Hide file tree

Showing 7 changed files with 65 additions and 14 deletions.
diff --git a/README.md b/README.md
@@ -26,6 +26,7 @@ If the callback is provided, it will also be called with an error or the results
 The supported options are the following:
 
 - `iterations`: The number of iterations to run for each test. Must be a positive number. The default is `10000`.
+- `errorThreshold`: If active, it stops the test run before the desider number of iterations if the standard error is below the provided value and at least 10% of the iterations have been run. Must be a number between `0` (which disables this option) and `100`. The default is `1`.
 - `print`: If print results on the console in a pretty tabular way. The default is `true`. It can be a boolean or a printing options object. The supported printing options are:
   - `colors`: If use colors. Default is `true`.
   - `compare`: If compare tests in the output. Default is `false`.

diff --git a/lib/index.js b/lib/index.js
@@ -22,6 +22,12 @@ function runIteration(context) {
         if (histogram.record(Number(process.hrtime.bigint() - start))) {
             context.current.records++;
         }
+        const standardError = histogram.stddev() / Math.sqrt(context.current.records) / histogram.mean();
+        if (context.errorThreshold &&
+            context.current.remaining / context.iterations < 0.9 &&
+            standardError < context.errorThreshold) {
+            context.current.remaining = 0;
+        }
         if (context.current.remaining === 0) {
             context.results[context.current.name] = {
                 success: true,
@@ -36,7 +42,7 @@ function runIteration(context) {
                     accu[percentile] = value;
                     return accu;
                 }, {}),
-                standardError: histogram.stddev() / Math.sqrt(context.current.records)
+                standardError
             };
             schedule(() => processQueue(context));
             return;
@@ -73,7 +79,7 @@ function processQueue(context) {
     testContext.current = {
         name: next[0],
         test: next[1],
-        remaining: context.iterations,
+        remaining: context.iterations - 1,
         records: 0,
         histogram: new native_hdr_histogram_1.default(1, 1e9, 5)
     };
@@ -100,17 +106,23 @@ function cronometro(tests, options, callback) {
         };
     }
     // Parse and validate options
-    const { iterations, print } = { iterations: 1e4, print: true, ...options };
+    const { iterations, errorThreshold, print } = { iterations: 1e4, errorThreshold: 1, print: true, ...options };
     // tslint:disable-next-line strict-type-predicates
     if (typeof iterations !== 'number' || iterations < 1) {
         callback(new Error('The iterations option must be a positive number.'));
         return promise;
     }
+    // tslint:disable-next-line strict-type-predicates
+    if (typeof errorThreshold !== 'number' || errorThreshold < 0 || errorThreshold > 100) {
+        callback(new Error('The errorThreshold option must be a number between 0 and 100.'));
+        return promise;
+    }
     // Process all tests
     const context = {
         queue: Object.entries(tests),
         results: {},
         iterations,
+        errorThreshold: errorThreshold / 100,
         callback(error, results) {
             if (error) {
                 callback(error);

diff --git a/lib/print.js b/lib/print.js
@@ -15,7 +15,7 @@ function printResults(results, colors, compare, mode) {
         if (!result.success) {
             return { name, error: result.error, throughput: '', standardError: '', relative: '', compared: '' };
         }
-        const { mean, standardError } = result;
+        const { size, mean, standardError } = result;
         const relative = last !== 0 ? (last / mean - 1) * 100 : 0;
         if (mode === 'base') {
             if (last === 0) {
@@ -27,10 +27,11 @@ function printResults(results, colors, compare, mode) {
             last = mean;
             compared = name;
         }
-        const standardErrorString = ((standardError / mean) * 100).toFixed(2);
+        const standardErrorString = (standardError * 100).toFixed(2);
         standardErrorPadding = Math.max(standardErrorPadding, standardErrorString.length);
         return {
             name,
+            size: size,
             error: null,
             throughput: (1e9 / mean).toFixed(2),
             standardError: standardErrorString,
@@ -41,15 +42,21 @@ function printResults(results, colors, compare, mode) {
     let currentColor = 0;
     const rows = entries.map((entry) => {
         if (entry.error) {
-            const row = [styler(`{{gray}}${entry.name}{{-}}`), styler('{{gray}}Errored{{-}}'), styler('{{gray}}N/A{{-}}')];
+            const row = [
+                styler(`{{gray}}${entry.name}{{-}}`),
+                styler(`{{gray}}${entry.size}{{-}}`),
+                styler('{{gray}}Errored{{-}}'),
+                styler('{{gray}}N/A{{-}}')
+            ];
             if (compare) {
                 row.push(styler('{{gray}}N/A{{-}}'));
             }
         }
-        const { name, throughput, standardError, relative } = entry;
+        const { name, size, throughput, standardError, relative } = entry;
         const color = styles[currentColor++ % styles.length];
         const row = [
             styler(`{{${color}}}${name}{{-}}`),
+            styler(`{{${color}}}${size}{{-}}`),
             styler(`{{${color}}}${throughput} op/sec{{-}}`),
             styler(`{{gray}}± ${standardError.padStart(standardErrorPadding, ' ')} %{{-}}`)
         ];
@@ -66,11 +73,13 @@ function printResults(results, colors, compare, mode) {
     const compareHeader = `Difference with ${mode === 'base' ? compared : 'previous'}`;
     rows.unshift([
         styler('{{bold white}}Test{{-}}'),
+        styler('{{bold white}}Samples{{-}}'),
         styler('{{bold white}}Result{{-}}'),
         styler('{{bold white}}Tolerance{{-}}')
     ]);
     rows.splice(rows.length - 1, 0, [
         styler('{{bold white}}Fastest test{{-}}'),
+        styler('{{bold white}}Samples{{-}}'),
         styler('{{bold white}}Result{{-}}'),
         styler('{{bold white}}Tolerance{{-}}')
     ]);

diff --git a/src/index.ts b/src/index.ts
@@ -31,6 +31,16 @@ function runIteration(context: TestContext): void {
       context.current.records++
     }
 
+    const standardError = histogram.stddev() / Math.sqrt(context.current.records) / histogram.mean()
+
+    if (
+      context.errorThreshold &&
+      context.current.remaining / context.iterations < 0.9 &&
+      standardError < context.errorThreshold
+    ) {
+      context.current.remaining = 0
+    }
+
     if (context.current.remaining === 0) {
       context.results[context.current.name] = {
         success: true,
@@ -45,7 +55,7 @@ function runIteration(context: TestContext): void {
             accu[percentile] = value
             return accu
           }, {}),
-        standardError: histogram.stddev() / Math.sqrt(context.current.records)
+        standardError
       }
 
       schedule(() => processQueue(context))
@@ -89,7 +99,7 @@ function processQueue(context: Context): void {
   testContext.current = {
     name: next[0],
     test: next[1],
-    remaining: context.iterations,
+    remaining: context.iterations - 1,
     records: 0,
     histogram: new Histogram(1, 1e9, 5)
   }
@@ -127,19 +137,26 @@ export function cronometro(
   }
 
   // Parse and validate options
-  const { iterations, print } = { iterations: 1e4, print: true, ...options }
+  const { iterations, errorThreshold, print } = { iterations: 1e4, errorThreshold: 1, print: true, ...options }
 
   // tslint:disable-next-line strict-type-predicates
   if (typeof iterations !== 'number' || iterations < 1) {
     callback(new Error('The iterations option must be a positive number.'))
     return promise
   }
 
+  // tslint:disable-next-line strict-type-predicates
+  if (typeof errorThreshold !== 'number' || errorThreshold < 0 || errorThreshold > 100) {
+    callback(new Error('The errorThreshold option must be a number between 0 and 100.'))
+    return promise
+  }
+
   // Process all tests
   const context: Context = {
     queue: Object.entries(tests), // Convert tests to a easier to process [name, func] list,
     results: {},
     iterations,
+    errorThreshold: errorThreshold / 100,
     callback(error?: Error | null, results?: Results): void {
       if (error) {
         callback!(error)

diff --git a/src/models.ts b/src/models.ts
@@ -53,6 +53,7 @@ export interface Context {
   queue: Array<[string, Test]>
   results: Results
   iterations: number
+  errorThreshold: number
 }
 
 export interface TestContext extends Context {

diff --git a/src/print.ts b/src/print.ts
@@ -6,6 +6,7 @@ const styles = ['red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white', 'g
 
 interface PrintInfo {
   name: string
+  size: number
   error: Error | null
   throughput: string
   standardError: string
@@ -28,7 +29,7 @@ export function printResults(results: Results, colors: boolean, compare: boolean
         return { name, error: result.error, throughput: '', standardError: '', relative: '', compared: '' } as PrintInfo
       }
 
-      const { mean, standardError } = result
+      const { size, mean, standardError } = result
       const relative = last !== 0 ? (last / mean! - 1) * 100 : 0
 
       if (mode === 'base') {
@@ -41,11 +42,12 @@ export function printResults(results: Results, colors: boolean, compare: boolean
         compared = name
       }
 
-      const standardErrorString = ((standardError! / mean!) * 100).toFixed(2)
+      const standardErrorString = (standardError! * 100).toFixed(2)
       standardErrorPadding = Math.max(standardErrorPadding, standardErrorString.length)
 
       return {
         name,
+        size: size!,
         error: null,
         throughput: (1e9 / mean!).toFixed(2),
         standardError: standardErrorString,
@@ -58,18 +60,24 @@ export function printResults(results: Results, colors: boolean, compare: boolean
 
   const rows: Array<Array<string>> = entries.map((entry: PrintInfo) => {
     if (entry.error) {
-      const row = [styler(`{{gray}}${entry.name}{{-}}`), styler('{{gray}}Errored{{-}}'), styler('{{gray}}N/A{{-}}')]
+      const row = [
+        styler(`{{gray}}${entry.name}{{-}}`),
+        styler(`{{gray}}${entry.size}{{-}}`),
+        styler('{{gray}}Errored{{-}}'),
+        styler('{{gray}}N/A{{-}}')
+      ]
 
       if (compare) {
         row.push(styler('{{gray}}N/A{{-}}'))
       }
     }
 
-    const { name, throughput, standardError, relative } = entry
+    const { name, size, throughput, standardError, relative } = entry
     const color = styles[currentColor++ % styles.length]
 
     const row = [
       styler(`{{${color}}}${name}{{-}}`),
+      styler(`{{${color}}}${size}{{-}}`),
       styler(`{{${color}}}${throughput} op/sec{{-}}`),
       styler(`{{gray}}± ${standardError.padStart(standardErrorPadding, ' ')} %{{-}}`)
     ]
@@ -89,12 +97,14 @@ export function printResults(results: Results, colors: boolean, compare: boolean
 
   rows.unshift([
     styler('{{bold white}}Test{{-}}'),
+    styler('{{bold white}}Samples{{-}}'),
     styler('{{bold white}}Result{{-}}'),
     styler('{{bold white}}Tolerance{{-}}')
   ])
 
   rows.splice(rows.length - 1, 0, [
     styler('{{bold white}}Fastest test{{-}}'),
+    styler('{{bold white}}Samples{{-}}'),
     styler('{{bold white}}Result{{-}}'),
     styler('{{bold white}}Tolerance{{-}}')
   ])

diff --git a/types/models.d.ts b/types/models.d.ts
@@ -47,6 +47,7 @@ export interface Context {
     queue: Array<[string, Test]>;
     results: Results;
     iterations: number;
+    errorThreshold: number;
 }
 export interface TestContext extends Context {
     current: {