diff --git a/README.md b/README.md index 213ba58..4e26ec4 100644 --- a/README.md +++ b/README.md @@ -242,10 +242,44 @@ export default async (entry: Entry) => { }; ``` -> ✴ The plugin also supports files with `.js`, `.mjs` and `.cjs` extensions. - In this example, the `filter` function will only exclude entries in the HAR where the request body contains a JSON object with a password field. +You can also modify the entries before saving the HAR by using the `transform` option. This option should be set to a path to a module that exports a function, similar to the filter option, to change the Entry object as desired. + +```js +cy.recordHar({ transform: '../support/remove-sensitive-data.ts' }); +``` + +Here's a simple example of what the `remove-sensitive-data.ts` module might look like: + +```ts +import { Entry } from 'har-format'; + +const PASSWORD_REGEXP = /("password":\s*")\w+("\s*)/; + +export default async (entry: Entry) => { + try { + // Remove sensitive information from the request body + if (entry.request.postData?.text) { + entry.request.postData.text = entry.request.postData.text.replace( + PASSWORD_REGEXP, + '$1***$2' + ); + } + + return entry; + } catch { + return entry; + } +}; +``` + +The function should take an Entry object as a parameter and return the modified. + +In this example, the transform function will replace any occurrences of password with `***` in the request body of each entry. + +> ✴ The plugin also supports files with `.js`, `.mjs` and `.cjs` extensions. + By default, the path is relative to the spec folder. But by providing a `rootDir` it will look for the module in the provided directory: ```js diff --git a/cypress/e2e/record-har.cy.ts b/cypress/e2e/record-har.cy.ts index d2254b2..4138ce2 100644 --- a/cypress/e2e/record-har.cy.ts +++ b/cypress/e2e/record-har.cy.ts @@ -41,6 +41,24 @@ describe('Record HAR', () => { }) ); + ['.js', '.ts', '.cjs'].forEach(ext => + it(`transforms a request using the custom postprocessor from ${ext} file`, () => { + cy.recordHar({ transform: `../fixtures/transform${ext}` }); + + cy.get('a[href$=fetch]').click(); + + cy.saveHar({ waitForIdle: true }); + + cy.findHar() + .its('log.entries') + .should('contain.something.like', { + response: { + content: { text: /\{"items":\[/ } + } + }); + }) + ); + it('excludes a request by its path', () => { cy.recordHar({ excludePaths: [/^\/api\/products$/, '^\\/api\\/users$'] }); diff --git a/cypress/fixtures/filter.cjs b/cypress/fixtures/filter.cjs index e8574f0..1c782b8 100644 --- a/cypress/fixtures/filter.cjs +++ b/cypress/fixtures/filter.cjs @@ -1,6 +1,6 @@ -module.exports = async req => { +module.exports = async entry => { try { - return /\{"products":\[/.test(req.response.content.text ?? ''); + return /\{"products":\[/.test(entry.response.content.text ?? ''); } catch { return false; } diff --git a/cypress/fixtures/filter.js b/cypress/fixtures/filter.js index e8574f0..1c782b8 100644 --- a/cypress/fixtures/filter.js +++ b/cypress/fixtures/filter.js @@ -1,6 +1,6 @@ -module.exports = async req => { +module.exports = async entry => { try { - return /\{"products":\[/.test(req.response.content.text ?? ''); + return /\{"products":\[/.test(entry.response.content.text ?? ''); } catch { return false; } diff --git a/cypress/fixtures/filter.mjs b/cypress/fixtures/filter.mjs index 6cd5883..5184bb3 100644 --- a/cypress/fixtures/filter.mjs +++ b/cypress/fixtures/filter.mjs @@ -1,6 +1,6 @@ -export default async req => { +export default async entry => { try { - return /\{"products":\[/.test(req.response.content.text ?? ''); + return /\{"products":\[/.test(entry.response.content.text ?? ''); } catch { return false; } diff --git a/cypress/fixtures/filter.ts b/cypress/fixtures/filter.ts index 9f69574..52a500d 100644 --- a/cypress/fixtures/filter.ts +++ b/cypress/fixtures/filter.ts @@ -1,8 +1,8 @@ import { Entry } from 'har-format'; -export default async (req: Entry) => { +export default async (entry: Entry) => { try { - return /\{"products":\[/.test(req.response.content.text ?? ''); + return /\{"products":\[/.test(entry.response.content.text ?? ''); } catch { return false; } diff --git a/cypress/fixtures/transform.cjs b/cypress/fixtures/transform.cjs new file mode 100644 index 0000000..2b26c15 --- /dev/null +++ b/cypress/fixtures/transform.cjs @@ -0,0 +1,19 @@ +module.exports = async entry => { + try { + if (entry.response.content?.text) { + entry.response.content = { + ...entry.response.content, + text: entry.response.content.text.replace( + /"products":\s*(.+)/g, + `"items":$1` + ) + }; + } + + return entry; + } catch { + return entry; + } +}; + + diff --git a/cypress/fixtures/transform.js b/cypress/fixtures/transform.js new file mode 100644 index 0000000..d120b62 --- /dev/null +++ b/cypress/fixtures/transform.js @@ -0,0 +1,17 @@ +module.exports = async entry => { + try { + if (entry.response.content?.text) { + entry.response.content = { + ...entry.response.content, + text: entry.response.content.text.replace( + /"products":\s*(.+)/g, + `"items":$1` + ) + }; + } + + return entry; + } catch { + return entry; + } +}; diff --git a/cypress/fixtures/transform.mjs b/cypress/fixtures/transform.mjs new file mode 100644 index 0000000..705b031 --- /dev/null +++ b/cypress/fixtures/transform.mjs @@ -0,0 +1,17 @@ +export async entry => { + try { + if (entry.response.content?.text) { + entry.response.content = { + ...entry.response.content, + text: entry.response.content.text.replace( + /"products":\s*(.+)/g, + `"items":$1` + ) + }; + } + + return entry; + } catch { + return entry; + } +}; diff --git a/cypress/fixtures/transform.ts b/cypress/fixtures/transform.ts new file mode 100644 index 0000000..5db4b5b --- /dev/null +++ b/cypress/fixtures/transform.ts @@ -0,0 +1,19 @@ +import { Entry } from 'har-format'; + +export default async (entry: Entry) => { + try { + if (entry.response.content?.text) { + entry.response.content = { + ...entry.response.content, + text: entry.response.content.text.replace( + /"products":\s*(.+)/g, + `"items":$1` + ) + }; + } + + return entry; + } catch { + return entry; + } +}; diff --git a/src/Plugin.spec.ts b/src/Plugin.spec.ts index 6c515b8..78a3f73 100644 --- a/src/Plugin.spec.ts +++ b/src/Plugin.spec.ts @@ -1,7 +1,7 @@ import type { RecordOptions, SaveOptions } from './Plugin'; import { Plugin } from './Plugin'; -import { Logger } from './utils/Logger'; -import { FileManager } from './utils/FileManager'; +import type { Logger } from './utils/Logger'; +import type { FileManager } from './utils/FileManager'; import type { Observer, ObserverFactory, diff --git a/src/Plugin.ts b/src/Plugin.ts index 9a542ea..171a062 100644 --- a/src/Plugin.ts +++ b/src/Plugin.ts @@ -5,7 +5,8 @@ import type { HarExporterFactory, NetworkObserverOptions, Observer, - ObserverFactory + ObserverFactory, + HarExporterOptions } from './network'; import { HarBuilder, NetworkIdleMonitor, NetworkRequest } from './network'; import { ErrorUtils } from './utils/ErrorUtils'; @@ -25,10 +26,7 @@ export interface SaveOptions { waitForIdle?: boolean; } -export type RecordOptions = NetworkObserverOptions & { - rootDir: string; - filter?: string; -}; +export type RecordOptions = NetworkObserverOptions & HarExporterOptions; interface Addr { port: number; @@ -81,10 +79,7 @@ export class Plugin { ); } - this.exporter = await this.exporterFactory.create({ - predicatePath: options.filter, - rootDir: options.rootDir - }); + this.exporter = await this.exporterFactory.create(options); this._connection = this.connectionFactory.create({ ...this.addr, maxRetries: 20, diff --git a/src/cdp/DefaultNetwork.ts b/src/cdp/DefaultNetwork.ts index 37e5bac..35a878b 100644 --- a/src/cdp/DefaultNetwork.ts +++ b/src/cdp/DefaultNetwork.ts @@ -1,6 +1,6 @@ import type { Network, NetworkEvent } from '../network'; import { ErrorUtils } from '../utils/ErrorUtils'; -import { Logger } from '../utils/Logger'; +import type { Logger } from '../utils/Logger'; import { TARGET_OR_BROWSER_CLOSED, UNABLE_TO_ATTACH_TO_TARGET diff --git a/src/index.ts b/src/index.ts index 60c5d18..95012e6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -22,7 +22,7 @@ const plugin = new Plugin( FileManager.Instance, new DefaultConnectionFactory(Logger.Instance), new DefaultObserverFactory(Logger.Instance), - new DefaultHarExporterFactory(FileManager.Instance) + new DefaultHarExporterFactory(FileManager.Instance, Logger.Instance) ); export const install = (on: Cypress.PluginEvents): void => { diff --git a/src/network/DefaultHarExporter.spec.ts b/src/network/DefaultHarExporter.spec.ts index af8c9b4..66fc6f3 100644 --- a/src/network/DefaultHarExporter.spec.ts +++ b/src/network/DefaultHarExporter.spec.ts @@ -1,11 +1,14 @@ import { NetworkRequest } from './NetworkRequest'; import { DefaultHarExporter } from './DefaultHarExporter'; +import type { Logger } from '../utils/Logger'; +import type { DefaultHarExporterOptions } from './DefaultHarExporterOptions'; import { anyString, instance, match, mock, reset, + spy, verify, when } from 'ts-mockito'; @@ -15,31 +18,47 @@ import type { WriteStream } from 'fs'; import { EOL } from 'os'; describe('DefaultHarExporter', () => { - const buffer = mock(); + const streamMock = mock(); + const loggerMock = mock(); const networkRequest = new NetworkRequest( '1', 'https://example.com', 'https://example.com', '1' ); - const predicate: jest.Mock<(entry: Entry) => Promise | unknown> = - jest.fn<(entry: Entry) => Promise | unknown>(); + const predicate = jest.fn<(entry: Entry) => Promise | unknown>(); + const transform = jest.fn<(entry: Entry) => Promise | Entry>(); + let harExporter!: DefaultHarExporter; + let options!: DefaultHarExporterOptions; + let optionsSpy!: DefaultHarExporterOptions; beforeEach(() => { - harExporter = new DefaultHarExporter(instance(buffer), predicate); + options = {}; + optionsSpy = spy(options); + + harExporter = new DefaultHarExporter( + instance(loggerMock), + instance(streamMock), + options + ); }); afterEach(() => { predicate.mockRestore(); - reset(buffer); + transform.mockRestore(); + reset( + streamMock, + optionsSpy, + loggerMock + ); }); describe('path', () => { it('should return the path serializing buffer', () => { // arrange const expected = '/path/file'; - when(buffer.path).thenReturn(Buffer.from(expected)); + when(streamMock.path).thenReturn(Buffer.from(expected)); // act const result = harExporter.path; @@ -51,7 +70,7 @@ describe('DefaultHarExporter', () => { it('should return the path', () => { // arrange const expected = '/path/file'; - when(buffer.path).thenReturn(expected); + when(streamMock.path).thenReturn(expected); // act const result = harExporter.path; @@ -67,7 +86,7 @@ describe('DefaultHarExporter', () => { harExporter.end(); // assert - verify(buffer.end()).once(); + verify(streamMock.end()).once(); }); }); @@ -75,20 +94,22 @@ describe('DefaultHarExporter', () => { it('should write the entry to the buffer', async () => { // arrange // @ts-expect-error type mismatch - when(buffer.closed).thenReturn(false); + when(streamMock.closed).thenReturn(false); + when(optionsSpy.filter).thenReturn(predicate); predicate.mockReturnValue(false); // act await harExporter.write(networkRequest); // assert - verify(buffer.write(match(`${EOL}`))).once(); + verify(streamMock.write(match(`${EOL}`))).once(); }); it('should write the entry to the buffer if the predicate returns throws an error', async () => { // arrange // @ts-expect-error type mismatch - when(buffer.closed).thenReturn(false); + when(streamMock.closed).thenReturn(false); + when(optionsSpy.filter).thenReturn(predicate); predicate.mockReturnValue( Promise.reject(new Error('something went wrong')) ); @@ -97,33 +118,67 @@ describe('DefaultHarExporter', () => { await harExporter.write(networkRequest); // assert - verify(buffer.write(match(`${EOL}`))).once(); + verify(streamMock.write(match(`${EOL}`))).once(); + }); + + it('should transform the entry before writing to the buffer', async () => { + // arrange + const entry = { foo: 'bar' } as unknown as Entry; + const entryString = JSON.stringify(entry); + // @ts-expect-error type mismatch + when(streamMock.closed).thenReturn(false); + when(optionsSpy.transform).thenReturn(transform); + transform.mockReturnValue(Promise.resolve(entry)); + + // act + await harExporter.write(networkRequest); + + // assert + verify(streamMock.write(match(`${entryString}${EOL}`))).once(); + }); + + it('should skip the entry when the transformation is failed with an error', async () => { + // arrange + // @ts-expect-error type mismatch + when(streamMock.closed).thenReturn(false); + when(optionsSpy.transform).thenReturn(transform); + transform.mockReturnValue( + Promise.reject(new Error('Something went wrong.')) + ); + + // act + await harExporter.write(networkRequest); + + // assert + verify(streamMock.write(anyString())).never(); }); it('should not write the entry to the buffer if the predicate returns true', async () => { // arrange // @ts-expect-error type mismatch - when(buffer.closed).thenReturn(false); + when(streamMock.closed).thenReturn(false); + when(optionsSpy.filter).thenReturn(predicate); predicate.mockReturnValue(true); // act await harExporter.write(networkRequest); // assert - verify(buffer.write(anyString())).never(); + verify(streamMock.write(anyString())).never(); }); it('should not write the entry to the buffer if the buffer is closed', async () => { // arrange // @ts-expect-error type mismatch - when(buffer.closed).thenReturn(true); + when(streamMock.closed).thenReturn(true); + when(optionsSpy.filter).thenReturn(predicate); predicate.mockReturnValue(false); // act await harExporter.write(networkRequest); // assert - verify(buffer.write(anyString())).never(); + verify(streamMock.write(anyString())).never(); }); }); }); diff --git a/src/network/DefaultHarExporter.ts b/src/network/DefaultHarExporter.ts index 7293977..2754580 100644 --- a/src/network/DefaultHarExporter.ts +++ b/src/network/DefaultHarExporter.ts @@ -1,9 +1,17 @@ import { EntryBuilder } from './EntryBuilder'; import type { NetworkRequest } from './NetworkRequest'; import type { HarExporter } from './HarExporter'; +import type { Logger } from '../utils/Logger'; +import { ErrorUtils } from '../utils/ErrorUtils'; +import type { + DefaultHarExporterOptions, + Filter, + Transformer +} from './DefaultHarExporterOptions'; import type { Entry } from 'har-format'; import type { WriteStream } from 'fs'; import { EOL } from 'os'; +import { format } from 'util'; export class DefaultHarExporter implements HarExporter { get path(): string { @@ -12,36 +20,74 @@ export class DefaultHarExporter implements HarExporter { return Buffer.isBuffer(path) ? path.toString('utf-8') : path; } + private get filter(): Filter | undefined { + return this.options?.filter; + } + + private get transform(): Transformer | undefined { + return this.options?.transform; + } + constructor( + private readonly logger: Logger, private readonly buffer: WriteStream, - private readonly predicate?: (entry: Entry) => Promise | unknown + private readonly options?: DefaultHarExporterOptions ) {} public async write(networkRequest: NetworkRequest): Promise { const entry = await new EntryBuilder(networkRequest).build(); - if (await this.applyPredicate(entry)) { + if (await this.applyFilter(entry)) { return; } - const json = JSON.stringify(entry); + const json = await this.serializeEntry(entry); // @ts-expect-error type mismatch - if (!this.buffer.closed) { + if (!this.buffer.closed && json) { this.buffer.write(`${json}${EOL}`); } } + public async serializeEntry(entry: Entry): Promise { + try { + const result = + typeof this.transform === 'function' + ? await this.transform(entry) + : entry; + + return JSON.stringify(result); + } catch (e) { + const stack = ErrorUtils.isError(e) ? e.stack : e; + + this.logger.debug( + format(`The entry has been filtered out due to an error: %j`, entry) + ); + this.logger.err( + `The entry is missing as a result of an error in the 'transform' function. +s +The stack trace for this error is: +${stack}` + ); + + return undefined; + } + } + public end(): void { this.buffer.end(); } - private async applyPredicate(entry: Entry) { + private async applyFilter(entry: Entry): Promise { try { - return ( - typeof this.predicate === 'function' && (await this.predicate?.(entry)) + return typeof this.filter === 'function' && (await this.filter(entry)); + } catch (e) { + const message = ErrorUtils.isError(e) ? e.message : e; + + this.logger.debug( + `The operation has encountered an error while processing the entry. ${message}` ); - } catch { + return false; } } diff --git a/src/network/DefaultHarExporterFactory.spec.ts b/src/network/DefaultHarExporterFactory.spec.ts index 40963e1..87d3e5d 100644 --- a/src/network/DefaultHarExporterFactory.spec.ts +++ b/src/network/DefaultHarExporterFactory.spec.ts @@ -2,6 +2,7 @@ import type { FileManager } from '../utils/FileManager'; import { DefaultHarExporterFactory } from './DefaultHarExporterFactory'; import { DefaultHarExporter } from './DefaultHarExporter'; import { Loader } from '../utils/Loader'; +import type { Logger } from '../utils/Logger'; import { instance, mock, reset, spy, verify, when } from 'ts-mockito'; import { jest, @@ -31,31 +32,44 @@ const resolvableInstance = (m: T): T => describe('DefaultHarExporterFactory', () => { const fileManagerMock = mock(); const writeStreamMock = mock(); + const loggerMock = mock(); const loaderSpy = spy(Loader); const predicate = jest.fn(); + const transform = jest.fn(); let factory!: DefaultHarExporterFactory; beforeEach(() => { - factory = new DefaultHarExporterFactory(instance(fileManagerMock)); + factory = new DefaultHarExporterFactory( + instance(fileManagerMock), + instance(loggerMock) + ); }); - afterEach(() => - reset( + afterEach(() => { + transform.mockReset(); + predicate.mockReset(); + reset( fileManagerMock, writeStreamMock, - loaderSpy - ) - ); + loaderSpy, + loggerMock + ); + }); describe('create', () => { it('should create a HarExporter instance', async () => { // arrange - const options = { rootDir: '/root', predicatePath: 'predicate.js' }; + const options = { + rootDir: '/root', + filter: 'predicate.js', + transform: 'transform.js' + }; when(fileManagerMock.createTmpWriteStream()).thenResolve( resolvableInstance(writeStreamMock) ); when(loaderSpy.load('/root/predicate.js')).thenResolve(predicate); + when(loaderSpy.load('/root/transform.js')).thenResolve(transform); // act const result = await factory.create(options); @@ -63,9 +77,10 @@ describe('DefaultHarExporterFactory', () => { // assert expect(result).toBeInstanceOf(DefaultHarExporter); verify(loaderSpy.load('/root/predicate.js')).once(); + verify(loaderSpy.load('/root/transform.js')).once(); }); - it('should create a HarExporter instance without predicate', async () => { + it('should create a HarExporter instance without pre and post processors', async () => { // arrange const options = { rootDir: '/root' }; when(fileManagerMock.createTmpWriteStream()).thenResolve( diff --git a/src/network/DefaultHarExporterFactory.ts b/src/network/DefaultHarExporterFactory.ts index 6f219aa..66aaefc 100644 --- a/src/network/DefaultHarExporterFactory.ts +++ b/src/network/DefaultHarExporterFactory.ts @@ -5,27 +5,55 @@ import type { import type { HarExporter } from './HarExporter'; import { DefaultHarExporter } from './DefaultHarExporter'; import { Loader } from '../utils/Loader'; -import { FileManager } from '../utils/FileManager'; -import type { Entry } from 'har-format'; +import type { FileManager } from '../utils/FileManager'; +import type { Logger } from '../utils/Logger'; +import type { + DefaultHarExporterOptions, + Filter, + Transformer +} from './DefaultHarExporterOptions'; import { resolve } from 'path'; export class DefaultHarExporterFactory implements HarExporterFactory { - constructor(private readonly fileManager: FileManager) {} + constructor( + private readonly fileManager: FileManager, + private readonly logger: Logger + ) {} + + public async create(options: HarExporterOptions): Promise { + const settings = await this.createSettings(options); + const stream = await this.fileManager.createTmpWriteStream(); + + return new DefaultHarExporter(this.logger, stream, settings); + } + + private async createSettings({ + filter, + transform, + rootDir + }: HarExporterOptions) { + const [preProcessor, postProcessor]: (Filter | Transformer | undefined)[] = + await Promise.all( + [filter, transform].map(path => this.loadCustomProcessor(path, rootDir)) + ); + + return { + filter: preProcessor, + transform: postProcessor + } as DefaultHarExporterOptions; + } - public async create({ - rootDir, - predicatePath - }: HarExporterOptions): Promise { - let predicate: ((request: Entry) => unknown) | undefined; + private async loadCustomProcessor( + path: string | undefined, + rootDir: string + ): Promise { + let processor: T | undefined; - if (predicatePath) { - const absolutePath = resolve(rootDir, predicatePath); - predicate = await Loader.load(absolutePath); + if (path) { + const absolutePath = resolve(rootDir, path); + processor = await Loader.load(absolutePath); } - return new DefaultHarExporter( - await this.fileManager.createTmpWriteStream(), - predicate - ); + return processor; } } diff --git a/src/network/DefaultHarExporterOptions.ts b/src/network/DefaultHarExporterOptions.ts new file mode 100644 index 0000000..1b8f7b2 --- /dev/null +++ b/src/network/DefaultHarExporterOptions.ts @@ -0,0 +1,9 @@ +import type { Entry } from 'har-format'; + +export type Filter = (entry: Entry) => Promise | unknown; +export type Transformer = (entry: Entry) => Promise | Entry; + +export interface DefaultHarExporterOptions { + filter?: Filter; + transform?: Transformer; +} diff --git a/src/network/HarExporterFactory.ts b/src/network/HarExporterFactory.ts index 168080b..cb65f24 100644 --- a/src/network/HarExporterFactory.ts +++ b/src/network/HarExporterFactory.ts @@ -2,7 +2,8 @@ import type { HarExporter } from './HarExporter'; export interface HarExporterOptions { rootDir: string; - predicatePath?: string; + filter?: string; + transform?: string; } export interface HarExporterFactory { diff --git a/src/network/index.ts b/src/network/index.ts index a78e53f..75bc308 100644 --- a/src/network/index.ts +++ b/src/network/index.ts @@ -21,3 +21,4 @@ export { HarBuilder } from './HarBuilder'; export { NetworkIdleMonitor } from './NetworkIdleMonitor'; export { NetworkObserver } from './NetworkObserver'; export { NetworkRequest, WebSocketFrameType } from './NetworkRequest'; +export type { DefaultHarExporterOptions } from './DefaultHarExporterOptions';