feat(betterer ✨): add betterer ci command (#250)

* Added a new `betterer ci` command that throws when there is any change in the results file.
* This required quite a bit of refactoring, but it's for the better

BREAKING CHANGE: Changes to the public API

.betterer.results

@@ -57,7 +57,7 @@ exports[`new eslint rules`] = {
       [62, 4, 68, "Promises must be handled appropriately or explicitly marked as ignored with the \`void\` operator.", "280309533"],
       [63, 4, 71, "Promises must be handled appropriately or explicitly marked as ignored with the \`void\` operator.", "2936064845"]
     ],
-    "packages/extension/src/server/validator.ts:3834209490": [
+    "packages/extension/src/server/validator.ts:3523409524": [
       [48, 10, 90, "Promises must be handled appropriately or explicitly marked as ignored with the \`void\` operator.", "1821734533"],
       [94, 12, 94, "Promises must be handled appropriately or explicitly marked as ignored with the \`void\` operator.", "2449957056"]
     ]

.github/workflows/build.yaml

@@ -26,4 +26,4 @@ jobs:
         with:
           frozen-lockfile: true
       - name: Run build
-        run: yarn run build
+        run: yarn run build:ci

docs/corner/corner.html

@@ -3,14 +3,14 @@
     width="80"
     height="80"
     viewBox="0 0 250 250"
-    style="fill: rgb(40, 40, 40); color: #fff; position: fixed; top: 0; border: 0; right: 0;"
+    style="fill: rgb(40, 40, 40); color: #fff; position: fixed; top: 0; border: 0; right: 0"
     aria-hidden="true"
   >
     <path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"></path>
     <path
       d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2"
       fill="currentColor"
-      style="transform-origin: 130px 106px;"
+      style="transform-origin: 130px 106px"
       class="octo-arm"
     ></path>
     <path

docs/index.html

@@ -114,9 +114,7 @@ <h3 class="typography--hero" id="how">How does Betterer work?</h3>
           src="https://media.giphy.com/media/TLIg5hdyiWeBuzJphQ/giphy.mp4"
         ></video>
       </div>
-      <h3 class="typography--emphasised typography--hero">
-        Isn't that neat!?
-      </h3>
+      <h3 class="typography--emphasised typography--hero">Isn't that neat!?</h3>
     </main>
     <footer class="footer__container">
       <p class="footer__link">

goldens/api/@betterer/betterer.d.ts

@@ -1,10 +1,11 @@
-export declare function betterer(partialConfig?: BettererConfigPartial): Promise<BettererStats>;
+export declare function betterer(partialConfig?: BettererConfigPartial): Promise<BettererSummary>;
 export declare namespace betterer {
     var file: typeof import("./betterer").file;
     var watch: typeof import("./betterer").watch;
 }
 
 export declare type BettererConfig = {
+    allowDiff: boolean;
     configPaths: BettererConfigPaths;
     cwd: string;
     filters: BettererConfigFilters;
@@ -21,6 +22,7 @@ export declare type BettererConfigFilters = ReadonlyArray<RegExp>;
 export declare type BettererConfigIgnore = ReadonlyArray<string>;
 
 export declare type BettererConfigPartial = Partial<{
+    allowDiff: boolean;
     configPaths: BettererConfigPaths | string;
     cwd: string;
     filters: BettererConfigFilters | ReadonlyArray<string> | string;
@@ -40,7 +42,7 @@ export declare type BettererContext = {
 
 export declare type BettererDeserialise<DeserialisedType, SerialisedType = DeserialisedType> = (serialised: SerialisedType) => DeserialisedType;
 
-export declare type BettererDiff<DeserialisedType = unknown, DiffType = null> = {
+export declare type BettererDiff<DeserialisedType = unknown, DiffType = unknown> = {
     expected: DeserialisedType;
     result: DeserialisedType;
     diff: DiffType;
@@ -116,8 +118,8 @@ export declare type BettererPrinter<SerialisedType> = (serialised: SerialisedTyp
 
 export declare type BettererReporter = {
     contextStart?(context: BettererContext): void;
-    contextEnd?(context: BettererContext, stats: BettererStats): void;
-    contextError?(context: BettererContext, error: BettererError, printed: Array<string>): void;
+    contextEnd?(context: BettererContext, summary: BettererSummary): void;
+    contextError?(context: BettererContext, error: BettererError): void;
     runsStart?(runs: BettererRuns, files: BettererFilePaths): void;
     runsEnd?(runs: BettererRuns, files: BettererFilePaths): void;
     runStart?(run: BettererRun): void;
@@ -150,6 +152,8 @@ export declare type BettererRun = {
     readonly isWorse: boolean;
 };
 
+export declare type BettererRunNames = Array<string>;
+
 export declare type BettererRuns = ReadonlyArray<BettererRun>;
 
 export declare type BettererSerialise<DeserialisedType, SerialisedType = DeserialisedType> = (result: DeserialisedType) => SerialisedType;
@@ -159,21 +163,25 @@ export declare type BettererSerialiser<DeserialisedType, SerialisedType = Deseri
     deserialise: BettererDeserialise<DeserialisedType, SerialisedType>;
 };
 
-export declare type BettererStats = {
-    readonly better: BettererTestNames;
-    readonly completed: BettererTestNames;
-    readonly expired: BettererTestNames;
-    readonly failed: BettererTestNames;
-    readonly new: BettererTestNames;
-    readonly ran: BettererTestNames;
-    readonly same: BettererTestNames;
-    readonly obsolete: BettererTestNames;
-    readonly skipped: BettererTestNames;
-    readonly updated: BettererTestNames;
-    readonly worse: BettererTestNames;
+export declare type BettererSummary = {
+    readonly runs: BettererRuns;
+    readonly obsolete: BettererRunNames;
+    readonly result: string;
+    readonly expected: string | null;
+    readonly hasDiff: boolean;
+    readonly better: BettererRuns;
+    readonly completed: BettererRuns;
+    readonly expired: BettererRuns;
+    readonly failed: BettererRuns;
+    readonly new: BettererRuns;
+    readonly ran: BettererRuns;
+    readonly same: BettererRuns;
+    readonly skipped: BettererRuns;
+    readonly updated: BettererRuns;
+    readonly worse: BettererRuns;
 };
 
-export declare class BettererTest<DeserialisedType = unknown, SerialisedType = DeserialisedType, DiffType = null> extends BettererTestState {
+export declare class BettererTest<DeserialisedType = unknown, SerialisedType = DeserialisedType, DiffType = unknown> extends BettererTestState {
     readonly constraint: BettererTestConstraint<DeserialisedType>;
     readonly deadline: number;
     readonly differ?: BettererDiffer<DeserialisedType, DiffType>;
@@ -191,9 +199,7 @@ export declare type BettererTestFunction<DeserialisedType> = (run: BettererRun)
 
 export declare type BettererTestGoal<DeserialisedType> = (result: DeserialisedType) => MaybeAsync<boolean>;
 
-export declare type BettererTestNames = Array<string>;
-
-export declare type BettererTestOptions<DeserialisedType = unknown, SerialisedType = DeserialisedType, DiffType = null> = {
+export declare type BettererTestOptions<DeserialisedType = unknown, SerialisedType = DeserialisedType, DiffType = unknown> = {
     constraint: BettererTestConstraint<DeserialisedType>;
     deadline?: Date | string;
     goal?: DeserialisedType | BettererTestGoal<DeserialisedType>;
@@ -208,6 +214,10 @@ export declare type BettererWatcher = {
     onRun(handler: BettererWatchRunHandler): void;
 };
 
-export declare type BettererWatchRunHandler = (runs: BettererRuns) => void;
+export declare type BettererWatchRunHandler = (summary: BettererSummary) => void;
 
 export declare function config(partialConfig: BettererConfigPartial): void;
+
+export declare function file(filePath: string, partialConfig?: BettererConfigPartial): Promise<BettererSummary>;
+
+export declare function watch(partialConfig?: BettererConfigPartial): Promise<BettererWatcher>;

goldens/api/@betterer/cli.d.ts

@@ -22,10 +22,12 @@ export declare type BettererPackageJSON = {
     devDependencies: Record<string, string>;
 };
 
+export declare function ciΔ(cwd: string, argv: BettererCLIArguments): Promise<BettererSummary>;
+
 export declare function cliΔ(argv: BettererCLIArguments): void;
 
 export declare function initΔ(cwd: string, argv: BettererCLIArguments): Promise<void>;
 
-export declare function startΔ(cwd: string, argv: BettererCLIArguments): Promise<BettererStats>;
+export declare function startΔ(cwd: string, argv: BettererCLIArguments): Promise<BettererSummary>;
 
 export declare function watchΔ(cwd: string, argv: BettererCLIArguments): Promise<void>;

goldens/api/@betterer/logger.d.ts

@@ -8,6 +8,8 @@ export declare type BettererLoggerCodeInfo = {
     end: number;
 };
 
+export declare type BettererLoggerDiffOptions = DiffOptions;
+
 export declare type BettererLoggerMessages = ReadonlyArray<string>;
 
 export declare type BettererLoggerOverwriteDone = typeof logUpdate['done'];
@@ -18,6 +20,8 @@ export declare function codeΔ(codeInfo: BettererLoggerCodeInfo): void;
 
 export declare const debugΔ: BettererLogger;
 
+export declare function diffΔ(expected: unknown, result: unknown, options?: DiffOptions): void;
+
 export declare const errorΔ: BettererLogger;
 
 export declare const infoΔ: BettererLogger;

goldens/api/@betterer/reporter.d.ts

@@ -1,5 +1,3 @@
-export declare function contextErrorΔ(_: BettererContext, error: BettererError, printed: Array<string>): void;
-
 export declare function getTestsΔ(count: number): string;
 
 export declare function quoteΔ(str: string): string;

package.json

@@ -11,7 +11,9 @@
   "scripts": {
     "api": "yarn compile && lerna run api",
     "betterer": "node ./packages/cli/./bin/betterer",
+    "betterer:ci": "node ./packages/cli/./bin/betterer-ci",
     "build": "yarn compile && yarn format && yarn lint && yarn test && yarn test:api && yarn betterer",
+    "build:ci": "yarn compile && yarn format && yarn lint && yarn test && yarn test:api && yarn betterer:ci",
     "clean": "rimraf reports && lerna exec \"rimraf dist\" && rimraf ./**/tsconfig.tsbuildinfo && lerna exec \"rimraf node_modules\" && rimraf node_modules && yarn",
     "commit": "git cz",
     "compile": "tsc -b packages",

packages/betterer/package.json

@@ -36,7 +36,6 @@
     "find-up": "^5.0.0",
     "gitignore-globs": "^0.1.1",
     "globby": "^11.0.1",
-    "jest-diff": "^26.1.0",
     "lines-and-columns": "^1.1.6",
     "safe-string-literal": "^1.0.1",
     "ts-node": "^9.0.0",

packages/betterer/src/betterer.ts

@@ -1,31 +1,36 @@
 import { logErrorΔ } from '@betterer/errors';
 
 import { BettererConfig, BettererConfigPartial, createConfig } from './config';
-import { BettererContextΩ, BettererStats, BettererRuns } from './context';
+import { BettererContextΩ, BettererSummary } from './context';
 import { registerExtensions } from './register';
 import { loadReporters, DEFAULT_REPORTER, WATCH_REPORTER } from './reporters';
 import { parallel, serial } from './runner';
 import { BettererWatcherΩ, BettererWatcher } from './watcher';
 
-export function betterer(partialConfig?: BettererConfigPartial): Promise<BettererStats> {
+export function betterer(partialConfig?: BettererConfigPartial): Promise<BettererSummary> {
   return runContext(async (config) => {
     const reporter = loadReporters(config.reporters.length ? config.reporters : [DEFAULT_REPORTER]);
     const context = new BettererContextΩ(config, reporter);
-    await context.setup();
-    const runs = await serial(context);
-    const stats = await context.process(runs);
-    context.tearDown();
-    return stats;
+    try {
+      await context.setup();
+      const summary = await serial(context);
+      context.end();
+      await context.save();
+      return summary;
+    } catch (error) {
+      reporter.contextError(context, error);
+      throw error;
+    }
   }, partialConfig);
 }
 
-export async function file(filePath: string, partialConfig?: BettererConfigPartial): Promise<BettererRuns> {
+export async function file(filePath: string, partialConfig?: BettererConfigPartial): Promise<BettererSummary> {
   return runContext(async (config) => {
     const context = new BettererContextΩ(config);
     await context.setup();
-    const runs = await parallel(context, [filePath]);
-    context.tearDown();
-    return runs;
+    const summary = await parallel(context, [filePath]);
+    context.end();
+    return summary;
   }, partialConfig);
 }
 betterer.file = file;
@@ -34,12 +39,17 @@ export function watch(partialConfig?: BettererConfigPartial): Promise<BettererWa
   return runContext(async (config) => {
     const reporter = loadReporters(config.reporters.length ? config.reporters : [WATCH_REPORTER]);
     const context = new BettererContextΩ(config, reporter);
-    const watcher = new BettererWatcherΩ(context, async (filePaths) => {
-      await context.setup();
-      return parallel(context, filePaths);
-    });
-    await watcher.setup();
-    return watcher;
+    try {
+      const watcher = new BettererWatcherΩ(context, async (filePaths) => {
+        await context.setup();
+        return parallel(context, filePaths);
+      });
+      await watcher.setup();
+      return watcher;
+    } catch (error) {
+      reporter.contextError(context, error);
+      throw error;
+    }
   }, partialConfig);
 }
 betterer.watch = watch;
@@ -52,8 +62,8 @@ async function runContext<RunResult, RunFunction extends (config: BettererConfig
     const config = createConfig(partialConfig);
     registerExtensions(config);
     return await run(config);
-  } catch (e) {
-    logErrorΔ(e);
-    throw e;
+  } catch (error) {
+    logErrorΔ(error);
+    throw error;
   }
 }

packages/betterer/src/config/config.ts

@@ -13,6 +13,7 @@ export function config(partialConfig: BettererConfigPartial): void {
 
 export function createConfig(partialConfig: BettererConfigPartial = {}): BettererConfig {
   const relativeConfig = {
+    allowDiff: partialConfig.allowDiff ?? true,
     configPaths: toArray<string>(partialConfig.configPaths || baseConfig.configPaths || ['./.betterer']),
     resultsPath: partialConfig.resultsPath || baseConfig.resultsPath || './.betterer.results',
     filters: toRegExps(toArray<string | RegExp>(partialConfig.filters || baseConfig.filters)),

packages/betterer/src/config/types.ts

@@ -4,19 +4,8 @@ export type BettererConfigPaths = ReadonlyArray<string>;
 export type BettererConfigFilters = ReadonlyArray<RegExp>;
 export type BettererConfigIgnore = ReadonlyArray<string>;
 
-export type BettererConfigPartial = Partial<{
-  configPaths: BettererConfigPaths | string;
-  cwd: string;
-  filters: BettererConfigFilters | ReadonlyArray<string> | string;
-  ignores: BettererConfigIgnore | string;
-  reporters: BettererReporterNames;
-  resultsPath: string;
-  silent: boolean;
-  tsconfigPath: string;
-  update: boolean;
-}>;
-
 export type BettererConfig = {
+  allowDiff: boolean;
   configPaths: BettererConfigPaths;
   cwd: string;
   filters: BettererConfigFilters;
@@ -27,3 +16,16 @@ export type BettererConfig = {
   tsconfigPath: string | null;
   update: boolean;
 };
+
+export type BettererConfigPartial = Partial<{
+  allowDiff: boolean;
+  configPaths: BettererConfigPaths | string;
+  cwd: string;
+  filters: BettererConfigFilters | ReadonlyArray<string> | string;
+  ignores: BettererConfigIgnore | string;
+  reporters: BettererReporterNames;
+  resultsPath: string;
+  silent: boolean;
+  tsconfigPath: string;
+  update: boolean;
+}>;