From 1052eb9e51a52f47f8e688a13fc03b108f4d587e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Rzepecki?= Date: Tue, 1 Aug 2023 10:48:45 +0200 Subject: [PATCH] feat: Record test failure details in mocha Fixes #231 --- components/recorder-cli/mocha/index.mjs | 27 +- components/recorder-cli/mocha/index.test.mjs | 42 +- components/trace/appmap/index.mjs | 21 + components/trace/appmap/index.test.mjs | 460 ++++++++++--------- test/cases/mocha/main.test.mjs | 6 +- test/cases/mocha/mocha/suite-3.subset.yaml | 4 + test/cases/mocha/mocha/suite-4.subset.yaml | 8 + test/cases/mocha/spec.yaml | 1 + 8 files changed, 347 insertions(+), 222 deletions(-) create mode 100644 test/cases/mocha/mocha/suite-3.subset.yaml create mode 100644 test/cases/mocha/mocha/suite-4.subset.yaml create mode 100644 test/cases/mocha/spec.yaml diff --git a/components/recorder-cli/mocha/index.mjs b/components/recorder-cli/mocha/index.mjs index 2a3e9aba5..e07f91520 100644 --- a/components/recorder-cli/mocha/index.mjs +++ b/components/recorder-cli/mocha/index.mjs @@ -1,6 +1,9 @@ import process from "node:process"; import { hooks } from "../../../lib/node/mocha-hook.mjs"; -import { ExternalAppmapError } from "../../error/index.mjs"; +import { + ExternalAppmapError, + parseExceptionStack, +} from "../../error/index.mjs"; import { logInfo, logErrorWhen } from "../../log/index.mjs"; import { getUuid } from "../../uuid/index.mjs"; import { extendConfiguration } from "../../configuration/index.mjs"; @@ -26,8 +29,24 @@ import { closeSocket, addSocketListener, } from "../../socket/index.mjs"; +import { stringifyLocation } from "../../location/index.mjs"; + +const { undefined, parseInt, Boolean, Promise } = globalThis; -const { undefined, parseInt, Promise } = globalThis; +const buildTestFailure = (error) => { + if (!error?.name) { + return undefined; + } + const message = [error.name, error.message].filter(Boolean).join(": "); + const frame = parseExceptionStack(error)?.at(0); + if (!frame) { + return { message }; + } + return { + message, + location: stringifyLocation({ position: frame, url: frame.fileName }), + }; +}; // Accessing mocha version via the prototype is not documented but it seems stable enough. // Added in https://github.com/mochajs/mocha/pull/3535 @@ -140,9 +159,11 @@ export const recordAsync = (configuration) => { "No running mocha test case", ExternalAppmapError, ); + const passed = context.currentTest.state === "passed"; recordStopTrack(frontend, running.track, { type: "test", - passed: context.currentTest.state === "passed", + passed, + failure: passed ? undefined : buildTestFailure(context.currentTest.err), }); flush(); running = null; diff --git a/components/recorder-cli/mocha/index.test.mjs b/components/recorder-cli/mocha/index.test.mjs index 3f68ed545..124c3aaa7 100644 --- a/components/recorder-cli/mocha/index.test.mjs +++ b/components/recorder-cli/mocha/index.test.mjs @@ -10,7 +10,7 @@ import { } from "../../configuration/index.mjs"; import { recordAsync } from "./index.mjs"; -const { setImmediate } = globalThis; +const { Error, setImmediate, undefined } = globalThis; const getLastMockSocket = readGlobal("GET_LAST_MOCK_SOCKET"); const receiveMockSocket = readGlobal("RECEIVE_MOCK_SOCKET"); @@ -45,18 +45,48 @@ setImmediate(() => { receiveMockSocket(getLastMockSocket(), "0"); }); -await beforeEach.call({ - currentTest: { - parent: { - fullTitle: () => "full-title-1", +const startTest = () => + beforeEach.call({ + currentTest: { + parent: { + fullTitle: () => "full-title-1", + }, }, + }); + +await startTest(); + +await afterEach.call({ + currentTest: { + state: "passed", }, }); +await startTest(); + await afterEach.call({ currentTest: { - state: "passed", + state: "failed", + err: new Error("test error"), }, }); await afterAll(); + +const testFailure = async (error) => { + await startTest(); + return afterEach.call({ + currentTest: { + state: "failed", + err: error, + }, + }); +}; + +await testFailure(undefined); +await testFailure({ name: "TestError", message: "test error" }); +await testFailure({ + name: "TestError", + message: "test error", + stack: "TestError: test error\n at /app/foo-bar.js:4:6", +}); diff --git a/components/trace/appmap/index.mjs b/components/trace/appmap/index.mjs index 131abaf0a..6f1d181fa 100644 --- a/components/trace/appmap/index.mjs +++ b/components/trace/appmap/index.mjs @@ -9,6 +9,7 @@ import { logInfo, logInfoWhen, } from "../../log/index.mjs"; +import { toRelativeUrl } from "../../url/index.mjs"; import { validateAppmap } from "../../validate-appmap/index.mjs"; import { parseLocation } from "../../location/index.mjs"; import { @@ -22,11 +23,13 @@ import { orderEventArray } from "./ordering/index.mjs"; import { getOutputUrl } from "./output.mjs"; const { + Boolean, Map, Array: { from: toArray }, String, Math: { round }, RangeError, + URL, } = globalThis; const VERSION = "1.8.0"; @@ -181,6 +184,18 @@ const printApplyEventDistribution = (events, codebase) => { .join(""); }; +const makeFailure = ({ message, location }, { base }) => { + if (!location) { + return { message }; + } + const { + url, + position: { line }, + } = parseLocation(location); + const relative = toRelativeUrl(new URL(url, base), base); + return { message, location: [relative, line].filter(Boolean).join(":") }; +}; + export const compileTrace = (configuration, files, messages, termination) => { logDebug("Trace: %j", { configuration, files, messages, termination }); const errors = []; @@ -290,12 +305,18 @@ export const compileTrace = (configuration, files, messages, termination) => { } } /* c8 ignore stop */ + const appmap = { version: VERSION, metadata: compileMetadata(configuration, errors, termination), classMap: exportClassmap(codebase), events: digested_events, }; + + if (termination.failure) { + appmap.metadata.test_failure = makeFailure(termination.failure, codebase); + } + if (configuration.validate.appmap) { validateAppmap(appmap); } diff --git a/components/trace/appmap/index.test.mjs b/components/trace/appmap/index.test.mjs index fdf85b2aa..1809a414e 100644 --- a/components/trace/appmap/index.test.mjs +++ b/components/trace/appmap/index.test.mjs @@ -53,236 +53,272 @@ const location = stringifyLocation({ position: { line: 1, column: 0 }, }); -assertDeepEqual( - compileTrace( - configuration, - [ - { - url: "protocol://host/home/dirname/filename.js", - content: "function f (x) {}", - hash: "hash", +const files = [ + { + url: "protocol://host/home/dirname/filename.js", + content: "function f (x) {}", + hash: "hash", + }, +]; + +const events = [ + { + type: "event", + session: "session", + site: "begin", + tab: tabs.event1, + group: 0, + time: 0, + payload: { + type: "apply", + function: location, + this: { type: "string", print: "THIS-PRINT-1" }, + arguments: [{ type: "string", print: "ARG-PRINT-1" }], + }, + }, + { + type: "event", + session: "session", + site: "begin", + tab: tabs.event2, + group: 0, + time: 0, + payload: { + type: "apply", + function: location, + this: { type: "string", print: "this-print-2" }, + arguments: [{ type: "string", print: "arg-print-2" }], + }, + }, + { + type: "group", + session: "session", + group: 0, + child: 1, + description: "description", + }, + { + type: "event", + session: "session", + site: "end", + tab: tabs.event2, + time: 0, + group: 0, + payload: { + type: "return", + function: location, + result: { + type: "string", + print: "result-print-2", }, - ], - [ - { - type: "event", - session: "session", - site: "begin", - tab: tabs.event1, - group: 0, - time: 0, - payload: { - type: "apply", - function: location, - this: { type: "string", print: "THIS-PRINT-1" }, - arguments: [{ type: "string", print: "ARG-PRINT-1" }], - }, + }, + }, + { + type: "event", + session: "session", + site: "end", + tab: tabs.event1, + time: 0, + group: 0, + payload: { + type: "return", + function: location, + result: { + type: "string", + print: "result-print-1", }, - { - type: "event", - session: "session", - site: "begin", - tab: tabs.event2, - group: 0, - time: 0, - payload: { - type: "apply", - function: location, - this: { type: "string", print: "this-print-2" }, - arguments: [{ type: "string", print: "arg-print-2" }], - }, + }, + }, + { + type: "amend", + session: "session", + tab: tabs.event1, + site: "begin", + payload: { + type: "apply", + function: location, + this: { type: "string", print: "this-print-1" }, + arguments: [{ type: "string", print: "arg-print-1" }], + }, + }, + { + type: "error", + session: "session", + error: { + type: "number", + print: "123", + }, + }, +]; + +const expected = { + url: "protocol://host/base/appmap_dir/process/appmap_file.appmap.json", + content: { + version: "1.8.0", + metadata: { + name: "map-name", + app: "app-name", + labels: [], + language: { + name: "javascript", + version: "ES.Next", + engine: undefined, }, - { - type: "group", - session: "session", - group: 0, - child: 1, - description: "description", + frameworks: [], + client: { + name: "agent", + url: "https://github.com/applandinc/appmap-agent-js", + version: "1.2.3", }, - { - type: "event", - session: "session", - site: "end", - tab: tabs.event2, - time: 0, - group: 0, - payload: { - type: "return", - function: location, - result: { - type: "string", - print: "result-print-2", - }, - }, + recorder: { name: "process", type: "process" }, + recording: undefined, + git: undefined, + test_status: undefined, + exception: { + class: "number", + message: "123", }, + }, + classMap: [ { - type: "event", - session: "session", - site: "end", - tab: tabs.event1, - time: 0, - group: 0, - payload: { - type: "return", - function: location, - result: { - type: "string", - print: "result-print-1", + type: "package", + name: "dirname", + children: [ + { + type: "class", + name: "filename", + children: [ + { + type: "function", + name: "f", + location: "./dirname/filename.js:1", + static: false, + labels: [], + comment: null, + source: null, + }, + ], }, - }, - }, - { - type: "amend", - session: "session", - tab: tabs.event1, - site: "begin", - payload: { - type: "apply", - function: location, - this: { type: "string", print: "this-print-1" }, - arguments: [{ type: "string", print: "arg-print-1" }], - }, - }, - { - type: "error", - session: "session", - error: { - type: "number", - print: "123", - }, + ], }, ], - { type: "manual" }, - ), - { - url: "protocol://host/base/appmap_dir/process/appmap_file.appmap.json", - content: { - version: "1.8.0", - metadata: { - name: "map-name", - app: "app-name", - labels: [], - language: { - name: "javascript", - version: "ES.Next", - engine: undefined, - }, - frameworks: [], - client: { - name: "agent", - url: "https://github.com/applandinc/appmap-agent-js", - version: "1.2.3", - }, - recorder: { name: "process", type: "process" }, - recording: undefined, - git: undefined, - test_status: undefined, - exception: { - class: "number", - message: "123", - }, - }, - classMap: [ - { - type: "package", - name: "dirname", - children: [ - { - type: "class", - name: "filename", - children: [ - { - type: "function", - name: "f", - location: "./dirname/filename.js:1", - static: false, - labels: [], - comment: null, - source: null, - }, - ], - }, - ], - }, - ], - events: [ - { - id: 1, - event: "call", - thread_id: 0, - defined_class: "filename", - method_id: "f", - path: "./dirname/filename.js", - lineno: 1, - static: false, - receiver: { - name: "this", - class: "string", - object_id: undefined, - value: "this-print-1", - }, - parameters: [ - { - name: "x", - class: "string", - object_id: undefined, - value: "arg-print-1", - }, - ], + events: [ + { + id: 1, + event: "call", + thread_id: 0, + defined_class: "filename", + method_id: "f", + path: "./dirname/filename.js", + lineno: 1, + static: false, + receiver: { + name: "this", + class: "string", + object_id: undefined, + value: "this-print-1", }, - { - id: 2, - event: "call", - thread_id: 0, - defined_class: "filename", - method_id: "f", - path: "./dirname/filename.js", - lineno: 1, - static: false, - receiver: { - name: "this", + parameters: [ + { + name: "x", class: "string", object_id: undefined, - value: "this-print-2", + value: "arg-print-1", }, - parameters: [ - { - name: "x", - class: "string", - object_id: undefined, - value: "arg-print-2", - }, - ], + ], + }, + { + id: 2, + event: "call", + thread_id: 0, + defined_class: "filename", + method_id: "f", + path: "./dirname/filename.js", + lineno: 1, + static: false, + receiver: { + name: "this", + class: "string", + object_id: undefined, + value: "this-print-2", }, - { - id: 3, - event: "return", - thread_id: 0, - parent_id: 2, - elapsed: 0, - return_value: { - name: "return", + parameters: [ + { + name: "x", class: "string", object_id: undefined, - value: "result-print-2", + value: "arg-print-2", }, - exceptions: undefined, + ], + }, + { + id: 3, + event: "return", + thread_id: 0, + parent_id: 2, + elapsed: 0, + return_value: { + name: "return", + class: "string", + object_id: undefined, + value: "result-print-2", }, - { - id: 4, - event: "return", - thread_id: 0, - parent_id: 1, - elapsed: 0, - return_value: { - name: "return", - class: "string", - object_id: undefined, - value: "result-print-1", - }, - exceptions: undefined, + exceptions: undefined, + }, + { + id: 4, + event: "return", + thread_id: 0, + parent_id: 1, + elapsed: 0, + return_value: { + name: "return", + class: "string", + object_id: undefined, + value: "result-print-1", }, - ], - }, + exceptions: undefined, + }, + ], + }, +}; + +assertDeepEqual( + compileTrace(configuration, files, events, { type: "manual" }), + expected, +); + +const testFailure = (raw, resolved) => { + expected.content.metadata.test_failure = resolved; + assertDeepEqual( + compileTrace(configuration, files, events, { + type: "manual", + failure: raw, + }), + expected, + ); +}; + +testFailure( + { + message: "test failure message", + location: "protocol://host/home/dirname/filename.js:42:1", + }, + { message: "test failure message", location: "dirname/filename.js:42" }, +); + +testFailure( + { message: "test failure message" }, + { message: "test failure message" }, +); + +testFailure( + { + message: "test failure message", + location: "/foo/bar/dirname/filename.js:42:7", + }, + { + message: "test failure message", + location: "../foo/bar/dirname/filename.js:42", }, ); diff --git a/test/cases/mocha/main.test.mjs b/test/cases/mocha/main.test.mjs index d98879c9b..80e6cc6d0 100644 --- a/test/cases/mocha/main.test.mjs +++ b/test/cases/mocha/main.test.mjs @@ -1,7 +1,7 @@ import { strict as Assert } from "node:assert"; import { describe, it } from "mocha"; import { main, mainAsync } from "./main.mjs"; -const { setTimeout } = globalThis; +const { Error, setTimeout } = globalThis; const { equal: assertEqual } = Assert; describe("suite", function () { it("main", function () { @@ -13,4 +13,8 @@ describe("suite", function () { it("mainAsync", async function () { assertEqual(await mainAsync(), "main"); }); + it("errors", () => { + throw new Error("test error"); + }); + it("fails", () => assertEqual("a", "b")); }); diff --git a/test/cases/mocha/mocha/suite-3.subset.yaml b/test/cases/mocha/mocha/suite-3.subset.yaml new file mode 100644 index 000000000..a228d7074 --- /dev/null +++ b/test/cases/mocha/mocha/suite-3.subset.yaml @@ -0,0 +1,4 @@ +metadata: + test_failure: + message: "Error: test error" + location: main.test.mjs:17 diff --git a/test/cases/mocha/mocha/suite-4.subset.yaml b/test/cases/mocha/mocha/suite-4.subset.yaml new file mode 100644 index 000000000..408754e2a --- /dev/null +++ b/test/cases/mocha/mocha/suite-4.subset.yaml @@ -0,0 +1,8 @@ +metadata: + test_failure: + message: | + AssertionError: Expected values to be strictly equal: + + 'a' !== 'b' + + location: main.test.mjs:19 diff --git a/test/cases/mocha/spec.yaml b/test/cases/mocha/spec.yaml new file mode 100644 index 000000000..18226ca2e --- /dev/null +++ b/test/cases/mocha/spec.yaml @@ -0,0 +1 @@ +commands: ["node $1 || cd ."] # the test suite is expected to fail