feat(coverage ✨): add coverage test

.github/workflows/build.yaml

@@ -12,7 +12,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [12.x, 14.x, 16.x, 17.x]
+        node-version: [12.x, 14.x, 16.x]
         operating-system: [windows-latest, macOS-latest, ubuntu-latest]
 
     steps:

goldens/api/coverage.api.md

@@ -0,0 +1,42 @@
+## API Report File for "@betterer/coverage"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+
+import { BettererFileGlobs } from '@betterer/betterer';
+import { BettererFilePatterns } from '@betterer/betterer';
+import { BettererRun } from '@betterer/betterer';
+import { BettererTest } from '@betterer/betterer';
+
+// @public
+export type BettererCoverageDiff = BettererCoverageIssues;
+
+// @public
+export type BettererCoverageIssue = Record<BettererCoverageTypes, number>;
+
+// @public
+export type BettererCoverageIssues = {
+    [filePath: string]: BettererCoverageIssue;
+};
+
+// @public
+export class BettererCoverageTest extends BettererTest<BettererCoverageIssues, BettererCoverageIssues, BettererCoverageDiff> {
+    // Warning: (ae-forgotten-export) The symbol "BettererCoverageTestFunction" needs to be exported by the entry point index.d.ts
+    //
+    // @internal
+    constructor(test: BettererCoverageTestFunction, coverageSummaryPath?: string);
+    exclude(...excludePatterns: BettererFilePatterns): this;
+    include(...includePatterns: BettererFileGlobs): this;
+}
+
+// @public
+export type BettererCoverageTypes = 'lines' | 'statements' | 'functions' | 'branches';
+
+// @public
+export function coverage(coverageSummaryPath?: string): BettererCoverageTest;
+
+// @public
+export function coverageTotal(coverageSummaryPath?: string): BettererCoverageTest;
+
+```

jest.config.js

@@ -10,9 +10,10 @@ module.exports = {
     '!<rootDir>/packages/fixture/dist/**/*.js'
   ],
   coverageDirectory: '<rootDir>/reports/coverage',
+  coverageReporters: ['clover', 'json', 'json-summary', 'lcov', 'text'],
   preset: 'ts-jest',
   testEnvironment: 'node',
-  testMatch: ['**/*.spec.ts', '!**/*.e2e.spec.ts'],
+  testMatch: ['**/*.spec.ts', '!**/*.e2e.spec.ts', '!**/workers.spec.ts'],
   watchPathIgnorePatterns: ['<rootDir>/fixtures', '<rootDir>/packages/[^/]+/src'],
   modulePathIgnorePatterns: ['<rootDir>/packages/extension']
 };

jest.workers.config.js

@@ -0,0 +1,8 @@
+module.exports = {
+  setupFilesAfterEnv: ['./test/setup.ts'],
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  testRegex: '.*\\workers.spec\\.ts$',
+  watchPathIgnorePatterns: ['<rootDir>/fixtures', '<rootDir>/packages/[^/]+/src'],
+  modulePathIgnorePatterns: ['<rootDir>/packages/extension']
+};

lerna.json

@@ -3,6 +3,7 @@
     "packages/betterer",
     "packages/cli",
     "packages/constraints",
+    "packages/coverage",
     "packages/docgen",
     "packages/errors",
     "packages/eslint",

package.json

@@ -15,8 +15,8 @@
     "betterer": "node ./packages/cli/bin/betterer --cache",
     "betterer:ci": "node ./packages/cli/bin/betterer ci",
     "betterer:precommit": "node ./packages/cli/bin/betterer precommit --cache",
-    "build": "yarn compile:packages && yarn test:api && yarn test:dependencies && yarn test && yarn test:e2e",
-    "build:ci": "yarn compile:packages && yarn test:api && yarn test:dependencies && yarn format && yarn lint && yarn test && yarn betterer:ci",
+    "build": "yarn compile:packages && yarn test:api && yarn test:dependencies && yarn test && yarn test:workers && yarn test:e2e",
+    "build:ci": "yarn compile:packages && yarn test:api && yarn test:dependencies && yarn format && yarn lint && yarn test && yarn test:workers && yarn betterer:ci",
     "build:extension": "lerna run build:extension",
     "build:publish": "yarn compile:packages && yarn test:api && yarn test:dependencies && yarn test",
     "clean": "yarn clean:tests && yarn clean:compile && yarn clean:modules",
@@ -38,6 +38,7 @@
     "test:dependencies": "yarn compile:dependencies-test && node ./test/dependencies/dist",
     "test:debug": "yarn clean:tests && WORKER_REQUIRE=\"false\" NODE_OPTIONS=\"--inspect-brk\" node_modules/jest-cli/bin/jest.js --runInBand --collectCoverage=false",
     "test:e2e": "yarn clean:tests && jest --runInBand --config ./jest.e2e.config.js",
+    "test:workers": "yarn clean:tests && jest --runInBand --config ./jest.workers.config.js",
     "test:extension": "lerna run test:extension",
     "prepublishOnly": "yarn run build:publish",
     "publish": "lerna publish --conventional-commits --no-commit-hooks --yes",

packages/betterer/src/test/test.ts

@@ -109,5 +109,16 @@ export class BettererTest<DeserialisedType, SerialisedType = DeserialisedType, D
 }
 
 export function isBettererTest(test: unknown): test is BettererTestBase {
-  return !!test && (test as BettererTestBase).constructor.name === BettererTest.name;
+  if (!test) {
+    return false;
+  }
+  return getBaseName((test as ObjectConstructor).constructor) === BettererTest.name;
+}
+
+function getBaseName(input: unknown): string | null {
+  const proto: unknown = Object.getPrototypeOf(input);
+  if (proto === Function.prototype) {
+    return (input as ObjectConstructor)?.name || null;
+  }
+  return getBaseName(proto);
 }

packages/coverage/README.md

@@ -0,0 +1,9 @@
+[![Betterer](https://raw.githubusercontent.com/phenomnomnominal/betterer/master/website/static/img/header.png)](https://phenomnomnominal.github.io/betterer/)
+
+# `@betterer/coverage`
+
+[![npm version](https://img.shields.io/npm/v/@betterer/coverage.svg)](https://www.npmjs.com/package/@betterer/coverage)
+
+Code coverage test for [**`Betterer`**](https://github.com/phenomnomnominal/betterer).
+
+[Check out the docs at `phenomnomnominal.github.io/betterer`! ?](https://phenomnomnominal.github.io/betterer/docs/coverage-test)

packages/coverage/api-extractor.json

@@ -0,0 +1,4 @@
+{
+  "extends": "../../api-extractor.json",
+  "mainEntryPointFilePath": "./dist/index.d.ts"
+}

packages/coverage/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "@betterer/coverage",
+  "description": "A betterer test which ensures that test coverage increases over time",
+  "version": "5.1.7",
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public"
+  },
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "author": "Lucas Treffenstädt <[email protected]>",
+  "homepage": "https://phenomnomnominal.github.io/betterer",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/phenomnomnominal/betterer.git"
+  },
+  "bugs": {
+    "url": "https://github.com/phenomnomnominal/betterer/issues"
+  },
+  "scripts": {
+    "compile": "tsc -b .",
+    "api": "api-extractor run --local --verbose"
+  },
+  "engines": {
+    "node": ">=12"
+  },
+  "dependencies": {
+    "@betterer/betterer": "^5.1.7",
+    "@betterer/constraints": "^5.0.0",
+    "@betterer/errors": "^5.0.0",
+    "@betterer/logger": "^5.1.5",
+    "minimatch": "^5.0.1",
+    "tslib": "^2.3.1"
+  },
+  "devDependencies": {
+    "@types/istanbul-lib-coverage": "2.0.4",
+    "@types/minimatch": "^3.0.5"
+  }
+}

packages/coverage/src/constraint.ts

@@ -0,0 +1,29 @@
+import { BettererConstraintResult } from '@betterer/constraints';
+
+import { differ } from './differ';
+import { BettererCoverageDiff, BettererCoverageIssues } from './types';
+
+export function constraint(result: BettererCoverageIssues, expected: BettererCoverageIssues): BettererConstraintResult {
+  const { diff } = differ(result, expected);
+  if (isWorse(diff)) {
+    return BettererConstraintResult.worse;
+  }
+  if (isBetter(diff)) {
+    return BettererConstraintResult.better;
+  }
+  return BettererConstraintResult.same;
+}
+
+function isWorse(diff: BettererCoverageDiff): boolean {
+  return Object.keys(diff).some((filePath) => {
+    const { lines, statements, functions, branches } = diff[filePath];
+    return lines < 0 || statements < 0 || functions < 0 || branches < 0;
+  });
+}
+
+function isBetter(diff: BettererCoverageDiff): boolean {
+  return Object.keys(diff).some((filePath) => {
+    const { lines, statements, functions, branches } = diff[filePath];
+    return lines > 0 || statements > 0 || functions > 0 || branches > 0;
+  });
+}

packages/coverage/src/coverage-test.ts

@@ -0,0 +1,69 @@
+import { BettererFileGlobs, BettererFilePatterns, BettererRun, BettererTest } from '@betterer/betterer';
+
+import { constraint } from './constraint';
+import { differ } from './differ';
+import { goal } from './goal';
+import { deserialise, serialise } from './serialiser';
+import { BettererCoverageDiff, BettererCoverageIssues, BettererCoverageTestFunction } from './types';
+import { flatten } from './utils';
+
+/**
+ * @public `BettererCoverageTest` provides a wrapper around {@link @betterer/betterer#BettererTest | `BettererTest` }
+ * that makes it easier to implement coverage tests.
+ *
+ * @remarks `BettererCoverageTest` provides a useful example for the more complex possibilities of the {@link @betterer/betterer#BettererTestOptions | `BettererTestOptions` }
+ * interface.
+ */
+export class BettererCoverageTest extends BettererTest<
+  BettererCoverageIssues,
+  BettererCoverageIssues,
+  BettererCoverageDiff
+> {
+  private _excluded: Array<RegExp> = [];
+  private _included: Array<string> = [];
+
+  /**
+   * @internal This could change at any point! Please don't use!
+   *
+   * You should not construct a `BettererCoverageTest` directly.
+   */
+  constructor(test: BettererCoverageTestFunction, coverageSummaryPath = './coverage/coverage-summary.json') {
+    super({
+      test: (run: BettererRun) => test(run, coverageSummaryPath, this._included, this._excluded),
+      constraint,
+      differ,
+      goal,
+      serialiser: {
+        serialise,
+        deserialise
+      }
+    });
+  }
+
+  /**
+   * Add a list of {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions | Regular Expression } filters for files to exclude when running the test.
+   *
+   * @param excludePatterns - RegExp filters to match file paths that should be excluded.
+   * @returns This {@link @betterer/coverage#BettererCoverageTest | `BettererCoverageTest`}, so it is chainable.
+   */
+  public exclude(...excludePatterns: BettererFilePatterns): this {
+    if (!this._included.length) {
+      this.include('**/*');
+    }
+    this._excluded = [...this._excluded, ...flatten(excludePatterns)];
+    return this;
+  }
+
+  /**
+   * Add a list of {@link https://www.npmjs.com/package/glob#user-content-glob-primer | glob }
+   * patterns for files to include when running the test.
+   *
+   * @param includePatterns - Glob patterns to match file paths that should be included. All
+   * `includes` should be relative to the {@link https://phenomnomnominal.github.io/betterer/docs/test-definition-file | test definition file}.
+   * @returns This {@link @betterer/coverage#BettererCoverageTest | `BettererCoverageTest`}, so it is chainable.
+   */
+  public include(...includePatterns: BettererFileGlobs): this {
+    this._included = [...this._included, ...flatten(includePatterns)];
+    return this;
+  }
+}

packages/coverage/src/coverage.ts

@@ -0,0 +1,24 @@
+import { BettererCoverageTest } from './coverage-test';
+import { test, testTotal } from './test';
+
+/**
+ * @public
+ * Use this test to track your per-file test coverage. Reads a {@link https://github.com/istanbuljs/istanbuljs/blob/master/packages/istanbul-reports/lib/json-summary/index.js | json-summary format}
+ * coverage summary. Make sure to run your tests separately before running Betterer.
+ *
+ * @param coverageSummaryPath - relative path to the coverage summary. Defaults to './coverage/coverage-summary.json'.
+ */
+export function coverage(coverageSummaryPath?: string): BettererCoverageTest {
+  return new BettererCoverageTest(test, coverageSummaryPath);
+}
+
+/**
+ * @public
+ * Use this test to track your total test coverage. Reads a {@link https://github.com/istanbuljs/istanbuljs/blob/master/packages/istanbul-reports/lib/json-summary/index.js | json-summary format}
+ * coverage summary. Make sure to run your tests separately before running Betterer.
+ *
+ * @param coverageSummaryPath - relative path to the coverage summary. Defaults to './coverage/coverage-summary.json'.
+ */
+export function coverageTotal(coverageSummaryPath?: string): BettererCoverageTest {
+  return new BettererCoverageTest(testTotal, coverageSummaryPath);
+}

packages/coverage/src/differ.ts

@@ -0,0 +1,89 @@
+import { BettererDiff } from '@betterer/betterer';
+import { BettererLog, BettererLogs } from '@betterer/logger';
+import { BettererCoverageDiff, BettererCoverageIssue, BettererCoverageIssues } from './types';
+
+export function differ(
+  expected: BettererCoverageIssues,
+  result: BettererCoverageIssues
+): BettererDiff<BettererCoverageDiff> {
+  const logs: BettererLogs = [];
+  const diff: BettererCoverageDiff = {
+    ...detectNewOrUpdatedFiles(expected, result, logs),
+    ...detectRemovedFiles(expected, result, logs)
+  };
+  return {
+    diff,
+    logs
+  };
+}
+
+function detectNewOrUpdatedFiles(
+  expected: BettererCoverageIssues,
+  result: BettererCoverageIssues,
+  logs: Array<BettererLog>
+) {
+  const diff: BettererCoverageDiff = {};
+  Object.keys(result).forEach((filePath) => {
+    const issue = result[filePath];
+    if (expected[filePath]) {
+      diff[filePath] = diffFileIssue(expected[filePath], issue, logs, filePath);
+    } else {
+      logs.push({ debug: `new file: ${filePath}` });
+      diff[filePath] = issue;
+    }
+  });
+  return diff;
+}
+
+function diffFileIssue(
+  expected: BettererCoverageIssue,
+  result: BettererCoverageIssue,
+  logs: Array<BettererLog>,
+  filePath: string
+): BettererCoverageIssue {
+  const fileDiff: BettererCoverageIssue = {
+    branches: 0,
+    functions: 0,
+    lines: 0,
+    statements: 0
+  };
+  const coverageTypes = Object.keys(fileDiff) as Array<keyof BettererCoverageIssue>;
+  coverageTypes.forEach((attribute: keyof BettererCoverageIssue) => {
+    const delta = result[attribute] - expected[attribute];
+    if (delta < 0) {
+      logs.push({
+        debug: `"${attribute}" coverage is better in "${filePath}": ${result[attribute]} < ${expected[attribute]}`
+      });
+    } else if (delta > 0) {
+      logs.push({
+        error: `"${attribute}" coverage is worse in "${filePath}": ${result[attribute]} > ${expected[attribute]}`
+      });
+    }
+    fileDiff[attribute] = delta;
+  });
+  return fileDiff;
+}
+
+function detectRemovedFiles(
+  expected: BettererCoverageIssues,
+  result: BettererCoverageIssues,
+  logs: Array<BettererLog>
+) {
+  const diff: BettererCoverageDiff = {};
+  Object.keys(expected).forEach((expectedFile) => {
+    if (!result[expectedFile]) {
+      logs.push({ debug: `${expectedFile} is gone.` });
+      diff[expectedFile] = getNegativeIssue(expected[expectedFile]);
+    }
+  });
+  return diff;
+}
+
+function getNegativeIssue(issue: BettererCoverageIssue): BettererCoverageIssue {
+  return {
+    branches: -issue.branches,
+    lines: -issue.lines,
+    functions: -issue.functions,
+    statements: -issue.statements
+  };
+}

packages/coverage/src/goal.ts

@@ -0,0 +1,8 @@
+import { BettererCoverageIssues } from './types';
+
+export function goal(result: BettererCoverageIssues): boolean {
+  return Object.keys(result).every((filePath) => {
+    const { lines, statements, functions, branches } = result[filePath];
+    return lines === 0 && statements === 0 && functions === 0 && branches === 0;
+  });
+}