feat(betterer ✨): run tests in parallel (#815)

goldens/api/@betterer/betterer.d.ts

@@ -15,6 +15,7 @@ export declare type BettererConfig = {
     resultsPath: string;
     silent: boolean;
     tsconfigPath: string | null;
+    workers: number;
     ci: boolean;
     precommit: boolean;
     strict: boolean;
@@ -92,7 +93,9 @@ export declare type BettererFileIssue = {
 
 export declare type BettererFileIssues = ReadonlyArray<BettererFileIssue>;
 
-export declare type BettererFilePaths = ReadonlyArray<string>;
+export declare type BettererFilePath = string;
+
+export declare type BettererFilePaths = ReadonlyArray<BettererFilePath>;
 
 export declare type BettererFilePatterns = ReadonlyArray<RegExp | ReadonlyArray<RegExp>>;
 
@@ -138,6 +141,7 @@ export declare type BettererOptionsBase = Partial<{
     resultsPath: string;
     silent: boolean;
     tsconfigPath: string;
+    workers: number;
 }>;
 
 export declare type BettererOptionsExcludes = Array<string | RegExp> | string;
@@ -228,6 +232,8 @@ export declare type BettererResult = {
 };
 
 export declare type BettererRun = {
+    readonly baseline: BettererResult | null;
+    readonly expected: BettererResult | null;
     readonly filePaths: BettererFilePaths | null;
     readonly name: string;
     readonly isNew: boolean;
@@ -249,12 +255,11 @@ export declare type BettererRuns = ReadonlyArray<BettererRun>;
 export declare type BettererRunSummaries = Array<BettererRunSummary>;
 
 export declare type BettererRunSummary = BettererRun & {
-    readonly diff: BettererDiff;
+    readonly diff: BettererDiff | null;
     readonly delta: BettererDelta | null;
-    readonly error: BettererError;
-    readonly expected: BettererResult;
+    readonly error: Error | null;
     readonly printed: string | null;
-    readonly result: BettererResult;
+    readonly result: BettererResult | null;
     readonly timestamp: number;
     readonly isBetter: boolean;
     readonly isComplete: boolean;
@@ -277,7 +282,7 @@ export declare type BettererSuite = {
     readonly runs: BettererRuns;
 };
 
-export declare type BettererSuiteSummaries = Array<BettererSuiteSummary>;
+export declare type BettererSuiteSummaries = ReadonlyArray<BettererSuiteSummary>;
 
 export declare type BettererSuiteSummary = BettererSuite & {
     readonly runs: BettererRunSummaries;
@@ -296,7 +301,7 @@ export declare type BettererSuiteSummary = BettererSuite & {
     readonly worse: BettererRunSummaries;
 };
 
-export declare class BettererTest<DeserialisedType, SerialisedType, DiffType> implements BettererTestBase<DeserialisedType, SerialisedType, DiffType> {
+export declare class BettererTest<DeserialisedType, SerialisedType = DeserialisedType, DiffType = null> implements BettererTestBase<DeserialisedType, SerialisedType, DiffType> {
     readonly config: BettererTestConfig<DeserialisedType, SerialisedType, DiffType>;
     get isOnly(): boolean;
     get isSkipped(): boolean;

goldens/api/@betterer/cli.d.ts

@@ -33,6 +33,7 @@ export declare type BettererCLIConfig = BettererCLIEnvConfig & {
     strict: boolean;
     tsconfig: string;
     update: boolean;
+    workers: number;
 };
 
 export declare type BettererCLIEnvConfig = {

packages/betterer/src/betterer.ts

@@ -1,28 +1,24 @@
 import { debug } from '@phenomnomnominal/debug';
 
 import { BettererOptionsRunner, BettererOptionsStart, BettererOptionsWatch } from './config';
-import { createGlobals } from './globals';
 import { BettererRunner, BettererRunnerΩ, BettererWatcherΩ } from './runner';
 import { BettererSuiteSummary } from './suite';
 
 export async function betterer(options: BettererOptionsStart = {}): Promise<BettererSuiteSummary> {
   initDebug();
-  const globals = await createGlobals(options);
-  const runner = new BettererRunnerΩ(globals);
-  return runner.run(globals.config.filePaths);
+  const runner = await BettererRunnerΩ.create(options);
+  return runner.run(runner.config.filePaths);
 }
 
 export async function runner(options: BettererOptionsRunner = {}): Promise<BettererRunner> {
   initDebug();
-  return new BettererRunnerΩ(await createGlobals(options));
+  return BettererRunnerΩ.create(options);
 }
 betterer.runner = runner;
 
 export async function watch(options: BettererOptionsWatch = {}): Promise<BettererRunner> {
   initDebug();
-  const watcher = new BettererWatcherΩ(await createGlobals({ ...options, watch: true }));
-  await watcher.setup();
-  return watcher;
+  return BettererWatcherΩ.create({ ...options, watch: true });
 }
 betterer.watch = watch;
 

packages/betterer/src/config/config.ts

@@ -1,10 +1,11 @@
 import { BettererError } from '@betterer/errors';
 import assert from 'assert';
 import { promises as fs } from 'fs';
+import * as os from 'os';
 import * as path from 'path';
 
 import { BettererFileResolverΩ, BettererVersionControlWorker } from '../fs';
-import { isBoolean, isRegExp, isString, isUndefined } from '../utils';
+import { isBoolean, isNumber, isRegExp, isString, isUndefined } from '../utils';
 import {
   BettererConfig,
   BettererConfigReporter,
@@ -14,6 +15,8 @@ import {
   BettererOptionsWatch
 } from './types';
 
+const TOTAL_CPUS = os.cpus().length;
+
 export async function createConfig(
   options: unknown = {},
   versionControl: BettererVersionControlWorker
@@ -36,6 +39,7 @@ export async function createConfig(
     resultsPath: baseOptions.resultsPath || './.betterer.results',
     silent: isDebug || baseOptions.silent || false,
     tsconfigPath: baseOptions.tsconfigPath || null,
+    workers: baseOptions.workers || Math.max(TOTAL_CPUS - 2, 1),
 
     // Runner:
     ignores: toArray<string>(runnerOptions.ignores),
@@ -85,6 +89,7 @@ function validateConfig(config: BettererConfig): void {
   validateStringRegExpArray('filters', config);
   validateString('resultsPath', config);
   validateBool('silent', config);
+  validateWorkers('workers', config);
 
   // Start:
   validateBool('ci', config);
@@ -100,6 +105,11 @@ function validateConfig(config: BettererConfig): void {
 }
 
 function overrideConfig(config: BettererConfig) {
+  // Silent mode:
+  if (config.silent) {
+    config.reporters = [];
+  }
+
   // CI mode:
   if (config.ci) {
     config.precommit = false;
@@ -147,6 +157,11 @@ function validateBool<Config, PropertyName extends keyof Config>(propertyName: P
   validate(isBoolean(value), `"${propertyName.toString()}" must be \`true\` or \`false\`. ${recieved(value)}`);
 }
 
+function validateNumber<Config, PropertyName extends keyof Config>(propertyName: PropertyName, config: Config): void {
+  const value = config[propertyName];
+  validate(isNumber(value), `"${propertyName.toString()}" must be a number. ${recieved(value)}`);
+}
+
 function validateString<Config, PropertyName extends keyof Config>(propertyName: PropertyName, config: Config): void {
   const value = config[propertyName];
   validate(isString(value), `"${propertyName.toString()}" must be a string. ${recieved(value)}`);
@@ -198,6 +213,17 @@ async function validateFilePath<Config, PropertyName extends keyof Config>(
   );
 }
 
+function validateWorkers<Config, PropertyName extends keyof Config>(propertyName: PropertyName, config: Config): void {
+  const value = config[propertyName];
+  validateNumber(propertyName, config);
+  validate(
+    isNumber(value) && value > 0 && value <= TOTAL_CPUS,
+    `"${propertyName.toString()}" must be more than zero and not more than the number of available CPUs (${TOTAL_CPUS}). ${recieved(
+      value
+    )}`
+  );
+}
+
 function validate(value: unknown, message: string): asserts value is boolean {
   // Wrap the AssertionError in a BettererError for logging:
   try {

packages/betterer/src/config/types.ts

@@ -18,6 +18,7 @@ export type BettererConfig = {
   resultsPath: string;
   silent: boolean;
   tsconfigPath: string | null;
+  workers: number;
   // Start:
   ci: boolean;
   precommit: boolean;
@@ -45,6 +46,7 @@ export type BettererOptionsBase = Partial<{
   resultsPath: string;
   silent: boolean;
   tsconfigPath: string;
+  workers: number;
 }>;
 
 export type BettererOptionsStartBase = BettererOptionsBase &

packages/betterer/src/context/context-summary.ts

@@ -1,16 +1,11 @@
 import { BettererConfig } from '../config';
-import { BettererSuiteSummaries, BettererSuiteSummary } from '../suite';
+import { BettererSuiteSummariesΩ, BettererSuiteSummaryΩ } from '../suite';
 import { BettererContextSummary } from './types';
-import { BettererGlobals } from '../types';
 
 export class BettererContextSummaryΩ implements BettererContextSummary {
-  public readonly config: BettererConfig;
+  constructor(public readonly config: BettererConfig, public readonly suites: BettererSuiteSummariesΩ) {}
 
-  constructor(private _globals: BettererGlobals, public readonly suites: BettererSuiteSummaries) {
-    this.config = this._globals.config;
-  }
-
-  public get lastSuite(): BettererSuiteSummary {
+  public get lastSuite(): BettererSuiteSummaryΩ {
     return this.suites[this.suites.length - 1];
   }
 }

packages/betterer/src/context/context.ts

@@ -4,21 +4,23 @@ import { BettererConfig } from '../config';
 import { BettererFilePaths, BettererVersionControlWorker } from '../fs';
 import { BettererReporterΩ } from '../reporters';
 import { BettererResultsΩ } from '../results';
+import { BettererRunWorkerPoolΩ, BettererRunΩ, createWorkerConfig } from '../run';
+import { BettererSuiteΩ, BettererSuiteSummariesΩ, BettererSuiteSummaryΩ } from '../suite';
+import { loadTestMeta } from '../test';
 import { defer } from '../utils';
-import { BettererSuiteSummaryΩ, BettererSuiteSummary, BettererSuiteSummaries, BettererSuiteΩ } from '../suite';
-import { BettererContext, BettererContextStarted, BettererContextSummary } from './types';
 import { BettererGlobals } from '../types';
 import { BettererContextSummaryΩ } from './context-summary';
+import { BettererContext, BettererContextStarted, BettererContextSummary } from './types';
 
-export class BettererContextΩ implements BettererContext, BettererGlobals {
+export class BettererContextΩ implements BettererContext {
   public readonly config: BettererConfig;
   public readonly reporter: BettererReporterΩ;
   public readonly results: BettererResultsΩ;
   public readonly versionControl: BettererVersionControlWorker;
 
-  private _suiteSummaries: BettererSuiteSummaries = [];
+  private _suiteSummaries: BettererSuiteSummariesΩ = [];
 
-  constructor(private _globals: BettererGlobals) {
+  constructor(private _globals: BettererGlobals, private _runWorkerPool: BettererRunWorkerPoolΩ) {
     this.config = this._globals.config;
     this.reporter = this._globals.reporter;
     this.results = this._globals.results;
@@ -27,19 +29,20 @@ export class BettererContextΩ implements BettererContext, BettererGlobals {
 
   public start(): BettererContextStarted {
     const contextLifecycle = defer<BettererContextSummary>();
+
     // Don't await here! A custom reporter could be awaiting
     // the lifecycle promise which is unresolved right now!
     const reportContextStart = this.reporter.contextStart(this, contextLifecycle.promise);
     return {
       end: async (): Promise<BettererContextSummary> => {
-        const contextSummary = new BettererContextSummaryΩ(this._globals, this._suiteSummaries);
+        const contextSummary = new BettererContextSummaryΩ(this.config, this._suiteSummaries);
         contextLifecycle.resolve(contextSummary);
         await reportContextStart;
         await this.reporter.contextEnd(contextSummary);
 
         await this.versionControl.writeCache();
 
-        const suiteSummaryΩ = contextSummary.lastSuite as BettererSuiteSummaryΩ;
+        const suiteSummaryΩ = contextSummary.lastSuite;
         if (suiteSummaryΩ.shouldWrite) {
           await this.results.write(suiteSummaryΩ.result);
           if (this.config.precommit) {
@@ -56,15 +59,26 @@ export class BettererContextΩ implements BettererContext, BettererGlobals {
     };
   }
 
-  public async run(filePaths: BettererFilePaths): Promise<BettererSuiteSummary> {
+  public async run(filePaths: BettererFilePaths): Promise<BettererSuiteSummaryΩ> {
     await this.results.sync();
     await this.versionControl.sync();
 
-    const validFilePaths = await this.versionControl.filterIgnored(filePaths);
+    filePaths = await this.versionControl.filterIgnored(filePaths);
+
+    const testMeta = loadTestMeta(this.config);
+    const testNames = Object.keys(testMeta);
+
+    const workerConfig = createWorkerConfig(this.config);
+
+    const runs = await Promise.all(
+      testNames.map(async (testName) => {
+        return BettererRunΩ.create(this._runWorkerPool, testName, workerConfig, filePaths, this.versionControl);
+      })
+    );
 
-    const suite = new BettererSuiteΩ(this, validFilePaths);
+    const suite = new BettererSuiteΩ(this, filePaths, runs);
     const suiteSummary = await suite.run();
-    this._suiteSummaries.push(suiteSummary);
+    this._suiteSummaries = [...this._suiteSummaries, suiteSummary];
     return suiteSummary;
   }
 }

packages/betterer/src/fs/file-resolver.ts

@@ -15,7 +15,6 @@ export class BettererFileResolverΩ implements BettererFileResolver {
   private _excluded: Array<RegExp> = [];
   private _included: Array<string> = [];
   private _includedResolved: Array<string> | null = null;
-
   private _validatedFilePaths: Array<string> = [];
   private _validatedFilePathsMap: Record<string, boolean> = {};
 

packages/betterer/src/fs/public.ts

@@ -1 +1,7 @@
-export { BettererFileGlobs, BettererFilePaths, BettererFilePatterns, BettererFileResolver } from './types';
+export {
+  BettererFileGlobs,
+  BettererFilePath,
+  BettererFilePaths,
+  BettererFilePatterns,
+  BettererFileResolver
+} from './types';

packages/betterer/src/fs/types.ts

@@ -1,7 +1,8 @@
 import { WorkerRequireModule, WorkerRequireModuleAsync } from '@phenomnomnominal/worker-require';
 
 export type BettererFileGlobs = ReadonlyArray<string | ReadonlyArray<string>>;
-export type BettererFilePaths = ReadonlyArray<string>;
+export type BettererFilePath = string;
+export type BettererFilePaths = ReadonlyArray<BettererFilePath>;
 export type BettererFilePatterns = ReadonlyArray<RegExp | ReadonlyArray<RegExp>>;
 
 export type BettererFileCacheMap = Record<string, string>;
@@ -22,7 +23,6 @@ export type BettererVersionControl = BettererFileCache & {
 };
 
 export type BettererVersionControlWorkerModule = WorkerRequireModule<typeof import('./version-control-worker')>;
-
 export type BettererVersionControlWorker =
   WorkerRequireModuleAsync<BettererVersionControlWorkerModule>['versionControl'];
 

packages/betterer/src/globals.ts

@@ -1,5 +1,5 @@
 import { createConfig } from './config';
-import { createVersionControl } from './fs';
+import { BettererVersionControlWorker, createVersionControl } from './fs';
 import { registerExtensions } from './register';
 import { DEFAULT_REPORTER, loadReporters } from './reporters';
 import { BettererResultsΩ } from './results';
@@ -28,3 +28,22 @@ export async function createGlobals(options: unknown = {}): Promise<BettererGlob
     throw error;
   }
 }
+
+export async function createWorkerGlobals(
+  options: unknown = {},
+  versionControl: BettererVersionControlWorker
+): Promise<BettererGlobals> {
+  const config = await createConfig(options, versionControl);
+  const { cache } = config;
+
+  if (cache) {
+    await versionControl.enableCache(config.cachePath);
+  }
+
+  await registerExtensions(config);
+
+  const reporter = loadReporters([]);
+  const results = new BettererResultsΩ(config);
+
+  return { config, reporter, results, versionControl };
+}

packages/betterer/src/index.ts

@@ -23,7 +23,13 @@ export {
   BettererOptionsWatch
 } from './config/public';
 export { BettererContext, BettererContextSummary, BettererDelta } from './context/public';
-export { BettererFileGlobs, BettererFilePaths, BettererFilePatterns, BettererFileResolver } from './fs/public';
+export {
+  BettererFileGlobs,
+  BettererFilePath,
+  BettererFilePaths,
+  BettererFilePatterns,
+  BettererFileResolver
+} from './fs/public';
 export { BettererResult } from './results/public';
 export { BettererReporter } from './reporters/public';
 export { BettererRun, BettererRunNames, BettererRunSummary, BettererRunSummaries, BettererRuns } from './run/public';

packages/betterer/src/results/result.ts

@@ -1,5 +1,4 @@
 import assert from 'assert';
-
 import { BettererResult } from './types';
 
 const NO_PREVIOUS_RESULT = Symbol('No Previous Result');