feat: record browser applications with man-in-the-middle proxy

REFERENCE.md

@@ -289,6 +289,7 @@ The agent filter code objects (functions or objects/classes) based on a format c
 * `socket "unix" | "net"` Defines the socket implementation to use: [`posix-socket`](https://www.npmjs.com/package/posix-socket) or [`net.Socket`](https://nodejs.org/api/net.html#class-netsocket). The `posix-socket` module provide synchronous methods which avoid creating asynchronous resources that are observed when `ordering` is `"causal"`. To avoid infinite loop, the `"net"` implementation buffers messages. The `"unix"` is probably the better option but it requires node-gyp compilation and is not available on windows. If `"unix"` is chosen but the `posix-socket` could not be installed, the agent will fallback on `"net"`. Default: `"unix"`.
 * `heartbeat <number> | null` Defines the interval in millisecond where the socket should be flushed. This only has effect if `socket` is `"net"`. Default: `1000`.
 * `threshold <number> | null` Defines the maximum number of message before the socket should be flushed. This only has effect if `socket` is `"net"`. Default: `100`.
+* `proxy-port <number> | null` Defines where the proxy should be listening to intercept http traffic and record html applications running on the browser. Use `0` for a random port. *Default*: `null` which does not deploy the proxy.
 * `trace-port <number> | <string>` Defines the communication port between frontend and backend. A string indicates a path to a unix domain socket which is faster. *Default*: `0` which will use a random available port.
 * `track-port <number> | <string>`: Port in the backend process for serving remote recording HTTP requests. *Default*: `0` A random port will be used.
 * `intercept-track-port <string>`: Regular expression to whitelist the ports in the frontend process for intercepting remote recording HTTP requests. *Default*: `"^"` Every detected HTTP ports will be spied upon.

component/build-prod.mjs

@@ -12,7 +12,12 @@ for (const [name, component, env, resolution] of [
     },
   ],
   [null, "error", "node", {}],
-  [null, "server", "node", { validate: "ajv", source: "default" }],
+  [
+    null,
+    "server",
+    "node",
+    { validate: "ajv", source: "default", instrumentation: "default" },
+  ],
   [null, "client", "node", { validate: "stub" }],
   [null, "init", "node", {}],
   [null, "status", "node", {}],

components/.ordering

@@ -64,6 +64,7 @@ recorder-standalone
 receptor
 command
 client
+mitm
 server
 questionnaire
 setup

components/configuration/default/index.mjs

@@ -307,6 +307,10 @@ const fields = {
     extend: overwrite,
     normalize: identity,
   },
+  "proxy-port": {
+    extend: overwrite,
+    normalize: identity,
+  },
   "trace-port": {
     extend: overwrite,
     normalize: normalizePort,
@@ -478,6 +482,7 @@ export const createConfiguration = (home) => ({
   host: "localhost",
   session: null,
   sessions: "^",
+  "proxy-port": null,
   "trace-port": 0, // possibly overwritten by the agent
   "http-switch": "__APPMAP__",
   "trace-protocol": "TCP",

components/mitm/default/.env

@@ -0,0 +1,2 @@
+test
+node

components/mitm/default/.ordering

@@ -0,0 +1,9 @@
+util.mjs
+stream.mjs
+html.mjs
+js.mjs
+forward.mjs
+intercept.mjs
+forge.mjs
+proxy.mjs
+index.mjs

components/mitm/default/forward.mjs

@@ -0,0 +1,100 @@
+import { Buffer } from "node:buffer";
+import { Socket as NetSocket } from "node:net";
+import { request as requestHttp } from "node:http";
+import { logWarning } from "../../log/index.mjs";
+
+const { from: toBuffer } = Buffer;
+
+export const forwardRequest = (host, inc_req, callback) => {
+  let done = false;
+  /* c8 ignore start */
+  inc_req.on("error", (error) => {
+    if (!done) {
+      done = true;
+      callback(error, null);
+    }
+  });
+  /* c8 ignore stop */
+  const out_req = requestHttp({
+    host: host.name,
+    port: host.port,
+    method: inc_req.method,
+    path: inc_req.url,
+    headers: inc_req.headers,
+  });
+  /* c8 ignore start */
+  out_req.on("error", (error) => {
+    if (!done) {
+      done = true;
+      callback(error, null);
+    }
+  });
+  /* c8 ignore stop */
+  out_req.on("response", (res) => {
+    if (!done) {
+      done = true;
+      callback(null, res);
+    }
+  });
+  inc_req.pipe(out_req);
+};
+
+export const forwardResponse = (inc_res, out_res) => {
+  out_res.writeHead(inc_res.statusCode, inc_res.statusMessage, inc_res.headers);
+  inc_res.pipe(out_res);
+};
+
+const stringifyHead = ({
+  method,
+  url,
+  httpVersion: version,
+  rawHeaders: headers,
+}) => {
+  const chunks = [];
+  chunks.push(`${method} ${url} HTTP/${version}`);
+  for (let index = 0; index < headers.length; index += 2) {
+    chunks.push(`${headers[index]}: ${headers[index + 1]}`);
+  }
+  chunks.push("");
+  chunks.push("");
+  return chunks.join("\r\n");
+};
+
+const forwardSpecial = (host, req, socket, head) => {
+  const forward_socket = new NetSocket();
+  /* c8 ignore start */
+  socket.on("error", (error) => {
+    logWarning(
+      "error on socket %j >> %s %s %j >> %o",
+      host,
+      req.method,
+      req.url,
+      req.headers,
+      error,
+    );
+    forward_socket.destroy();
+  });
+  forward_socket.on("error", (error) => {
+    logWarning(
+      "error on forward socket %j >> %s %s %j >> %o",
+      host,
+      req.method,
+      req.url,
+      req.headers,
+      error,
+    );
+    socket.destroy();
+  });
+  /* c8 ignore stop */
+  forward_socket.connect(host.port, host.name);
+  forward_socket.on("connect", () => {
+    forward_socket.write(toBuffer(stringifyHead(req), "utf8"));
+    forward_socket.write(head);
+    socket.pipe(forward_socket);
+    forward_socket.pipe(socket);
+  });
+};
+
+export const forwardUpgrade = forwardSpecial;
+
+export const forwardConnect = forwardSpecial;

components/mitm/default/forward.test.mjs

@@ -0,0 +1,173 @@
+import {
+  createServer as createHttpServer,
+  request as requestHttp,
+} from "node:http";
+import { Buffer } from "node:buffer";
+import { default as WebSocket, WebSocketServer } from "ws";
+import { assertEqual, assertDeepEqual } from "../../__fixture__.mjs";
+import { bufferReadable } from "./stream.mjs";
+import {
+  forwardRequest,
+  forwardResponse,
+  forwardUpgrade,
+  forwardConnect,
+} from "./forward.mjs";
+
+const {
+  String,
+  Promise,
+  JSON: { stringify: stringifyJSON, parse: parseJSON },
+} = globalThis;
+
+const { from: toBuffer } = Buffer;
+
+const wss = new WebSocketServer({ noServer: true });
+
+//////////////////
+// Setup Server //
+//////////////////
+
+const server = createHttpServer();
+
+server.on("request", (req, res) => {
+  bufferReadable(req, (buffer) => {
+    const body = toBuffer(
+      stringifyJSON({
+        method: req.method,
+        path: req.url,
+        version: req.httpVersion,
+        body: buffer.toString("utf8"),
+      }),
+      "utf8",
+    );
+    res.writeHead(200, {
+      "content-type": "application/json",
+      "content-length": body.length,
+    });
+    res.end(body);
+  });
+});
+
+server.on("upgrade", (req, socket, head) => {
+  wss.handleUpgrade(req, socket, head, (ws) => {
+    ws.on("message", (buffer) => {
+      ws.send(buffer);
+    });
+  });
+});
+
+server.listen(0);
+
+await new Promise((resolve, reject) => {
+  server.on("listening", resolve);
+  server.on("error", reject);
+});
+
+const host = { name: "localhost", port: server.address().port };
+
+/////////////////
+// Setup Forge //
+/////////////////
+
+const forge = createHttpServer();
+
+forge.on("request", (inc_req, out_res) => {
+  forwardRequest(host, inc_req, (error, inc_res) => {
+    assertEqual(error, null);
+    forwardResponse(inc_res, out_res);
+  });
+});
+
+forge.on("upgrade", (req, socket, head) => {
+  forwardUpgrade(host, req, socket, head);
+});
+
+forge.on("connect", (req, socket, head) => {
+  forwardConnect(host, req, socket, head);
+});
+
+forge.listen(0);
+
+await new Promise((resolve, reject) => {
+  forge.on("listening", resolve);
+  forge.on("error", reject);
+});
+
+/////////////
+// Request //
+/////////////
+
+{
+  const req = requestHttp({
+    host: "localhost",
+    port: forge.address().port,
+    method: "POST",
+    path: "/path",
+    version: "1.1",
+  });
+  req.end("body");
+  const res = await new Promise((resolve, reject) => {
+    req.on("response", resolve);
+    req.on("error", reject);
+  });
+  assertDeepEqual(
+    parseJSON(
+      (
+        await new Promise((resolve) => {
+          bufferReadable(res, resolve);
+        })
+      ).toString("utf8"),
+    ),
+    {
+      method: "POST",
+      path: "/path",
+      version: "1.1",
+      body: "body",
+    },
+  );
+}
+
+/////////////
+// Upgrade //
+/////////////
+
+{
+  const ws = new WebSocket(`ws://localhost:${String(forge.address().port)}`);
+  await new Promise((resolve, reject) => {
+    ws.on("open", resolve);
+    ws.on("error", reject);
+  });
+  ws.send(toBuffer("message", "utf8"));
+  assertEqual(
+    (
+      await new Promise((resolve, reject) => {
+        ws.on("message", resolve);
+        ws.on("error", reject);
+      })
+    ).toString("utf8"),
+    "message",
+  );
+  ws.close();
+  await new Promise((resolve, reject) => {
+    ws.on("close", resolve);
+    ws.on("error", reject);
+  });
+}
+
+//////////////
+// Teardown //
+//////////////
+
+forge.close();
+
+await new Promise((resolve, reject) => {
+  forge.on("error", reject);
+  forge.on("close", resolve);
+});
+
+server.close();
+
+await new Promise((resolve, reject) => {
+  server.on("error", reject);
+  server.on("close", resolve);
+});