Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: add accurate percentiles approximation computation #87

Merged
merged 15 commits into from
Oct 13, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 5 additions & 6 deletions src/task.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import Bench from './bench';
import tTable from './constants';
import { createBenchEvent } from './event';
import { AddEventListenerOptionsArgument, RemoveEventListenerOptionsArgument } from './types';
import { getVariance, isAsyncTask } from './utils';
import { getVariance, isAsyncTask, quantileSorted } from './utils';

/**
* A class that represents each benchmark task in Tinybench. It keeps track of the
Expand Down Expand Up @@ -152,11 +152,10 @@ export default class Task extends EventTarget {
const moe = sem * critical;
const rme = (moe / mean) * 100;

// mitata: https://github.com/evanwashere/mitata/blob/3730a784c9d83289b5627ddd961e3248088612aa/src/lib.mjs#L12
const p75 = samples[Math.ceil(samplesLength * 0.75) - 1]!;
const p99 = samples[Math.ceil(samplesLength * 0.99) - 1]!;
const p995 = samples[Math.ceil(samplesLength * 0.995) - 1]!;
const p999 = samples[Math.ceil(samplesLength * 0.999) - 1]!;
const p75 = quantileSorted(samples, 0.75);
const p99 = quantileSorted(samples, 0.99);
const p995 = quantileSorted(samples, 0.995);
const p999 = quantileSorted(samples, 0.999);

if (this.bench.signal?.aborted) {
return this;
Expand Down
65 changes: 57 additions & 8 deletions src/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,6 @@ function isPromiseLike<T>(maybePromiseLike: any): maybePromiseLike is PromiseLik
);
}

/**
* Computes the variance of a sample.
*/
export const getVariance = (samples: number[], mean: number) => {
const result = samples.reduce((sum, n) => sum + ((n - mean) ** 2), 0);
return result / (samples.length - 1) || 0;
};

// eslint-disable-next-line @typescript-eslint/no-empty-function
const AsyncFunctionConstructor = (async () => {}).constructor;

Expand Down Expand Up @@ -64,3 +56,60 @@ export const isAsyncTask = async (task: Task) => {
return false;
}
};

/**
* Computes the average of a sample.
*
* @param samples the sample
* @returns the average of the sample
*/
export const average = (samples: number[]) => {
if (samples.length === 0) {
throw new Error('samples must not be empty');
}

return samples.reduce((a, b) => a + b, 0) / samples.length || 0;
};

/**
* Computes the variance of a sample with Bessel's correction.
*
* @param samples the sample
* @param avg the average of the sample
* @returns the variance of the sample
*/
export const getVariance = (samples: number[], avg = average(samples)) => {
const result = samples.reduce((sum, n) => sum + ((n - avg) ** 2), 0);
return result / (samples.length - 1) || 0;
};

/**
* Computes the q-quantile of a sorted sample.
*
* @param samples the sorted sample
* @param q the quantile to compute
* @returns the q-quantile of the sample
*/
export const quantileSorted = (samples: number[], q: number) => {
if (samples.length === 0) {
throw new Error('samples must not be empty');
}
if (q < 0 || q > 1) {
throw new Error('q must be between 0 and 1');
}
if (q === 0) {
return samples[0];
}
if (q === 1) {
return samples[samples.length - 1];
}
const base = (samples.length - 1) * q;
const baseIndex = Math.floor(base);
if (samples[baseIndex + 1] != null) {
return (
(samples[baseIndex]!)
+ (base - baseIndex) * ((samples[baseIndex + 1]!) - samples[baseIndex]!)
);
}
return samples[baseIndex];
};
Loading