From 87b24474503f5b18697aca5514088af5b55e692a Mon Sep 17 00:00:00 2001 From: Artem Derevnjuk Date: Wed, 1 Feb 2023 15:20:37 +0400 Subject: [PATCH 1/9] feat(plugin): allow to transform entries before saving a HAR file closes #174 --- src/Plugin.spec.ts | 4 +- src/Plugin.ts | 6 +- src/cdp/DefaultNetwork.ts | 2 +- src/index.ts | 2 +- src/network/DefaultHarExporter.spec.ts | 87 +++++++++++++++---- src/network/DefaultHarExporter.ts | 59 +++++++++++-- src/network/DefaultHarExporterFactory.spec.ts | 30 +++++-- src/network/DefaultHarExporterFactory.ts | 57 ++++++++---- src/network/DefaultHarExporterOptions.ts | 9 ++ src/network/HarExporterFactory.ts | 1 + src/network/index.ts | 1 + 11 files changed, 206 insertions(+), 52 deletions(-) create mode 100644 src/network/DefaultHarExporterOptions.ts diff --git a/src/Plugin.spec.ts b/src/Plugin.spec.ts index 6c515b8..78a3f73 100644 --- a/src/Plugin.spec.ts +++ b/src/Plugin.spec.ts @@ -1,7 +1,7 @@ import type { RecordOptions, SaveOptions } from './Plugin'; import { Plugin } from './Plugin'; -import { Logger } from './utils/Logger'; -import { FileManager } from './utils/FileManager'; +import type { Logger } from './utils/Logger'; +import type { FileManager } from './utils/FileManager'; import type { Observer, ObserverFactory, diff --git a/src/Plugin.ts b/src/Plugin.ts index 9a542ea..28857eb 100644 --- a/src/Plugin.ts +++ b/src/Plugin.ts @@ -28,6 +28,7 @@ export interface SaveOptions { export type RecordOptions = NetworkObserverOptions & { rootDir: string; filter?: string; + transform?: string; }; interface Addr { @@ -81,10 +82,7 @@ export class Plugin { ); } - this.exporter = await this.exporterFactory.create({ - predicatePath: options.filter, - rootDir: options.rootDir - }); + this.exporter = await this.exporterFactory.create(options); this._connection = this.connectionFactory.create({ ...this.addr, maxRetries: 20, diff --git a/src/cdp/DefaultNetwork.ts b/src/cdp/DefaultNetwork.ts index 37e5bac..35a878b 100644 --- a/src/cdp/DefaultNetwork.ts +++ b/src/cdp/DefaultNetwork.ts @@ -1,6 +1,6 @@ import type { Network, NetworkEvent } from '../network'; import { ErrorUtils } from '../utils/ErrorUtils'; -import { Logger } from '../utils/Logger'; +import type { Logger } from '../utils/Logger'; import { TARGET_OR_BROWSER_CLOSED, UNABLE_TO_ATTACH_TO_TARGET diff --git a/src/index.ts b/src/index.ts index 60c5d18..95012e6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -22,7 +22,7 @@ const plugin = new Plugin( FileManager.Instance, new DefaultConnectionFactory(Logger.Instance), new DefaultObserverFactory(Logger.Instance), - new DefaultHarExporterFactory(FileManager.Instance) + new DefaultHarExporterFactory(FileManager.Instance, Logger.Instance) ); export const install = (on: Cypress.PluginEvents): void => { diff --git a/src/network/DefaultHarExporter.spec.ts b/src/network/DefaultHarExporter.spec.ts index af8c9b4..cc9fe2a 100644 --- a/src/network/DefaultHarExporter.spec.ts +++ b/src/network/DefaultHarExporter.spec.ts @@ -1,11 +1,14 @@ import { NetworkRequest } from './NetworkRequest'; import { DefaultHarExporter } from './DefaultHarExporter'; +import type { Logger } from '../utils/Logger'; +import type { DefaultHarExporterOptions } from './DefaultHarExporterOptions'; import { anyString, instance, match, mock, reset, + spy, verify, when } from 'ts-mockito'; @@ -15,31 +18,47 @@ import type { WriteStream } from 'fs'; import { EOL } from 'os'; describe('DefaultHarExporter', () => { - const buffer = mock(); + const streamMock = mock(); + const loggerMock = mock(); const networkRequest = new NetworkRequest( '1', 'https://example.com', 'https://example.com', '1' ); - const predicate: jest.Mock<(entry: Entry) => Promise | unknown> = - jest.fn<(entry: Entry) => Promise | unknown>(); + const predicate = jest.fn<(entry: Entry) => Promise | unknown>(); + const transform = jest.fn<(entry: Entry) => Promise | Entry>(); + let harExporter!: DefaultHarExporter; + let options!: DefaultHarExporterOptions; + let optionsSpy!: DefaultHarExporterOptions; beforeEach(() => { - harExporter = new DefaultHarExporter(instance(buffer), predicate); + options = {}; + optionsSpy = spy(options); + + harExporter = new DefaultHarExporter( + instance(loggerMock), + instance(streamMock), + options + ); }); afterEach(() => { predicate.mockRestore(); - reset(buffer); + transform.mockRestore(); + reset( + streamMock, + optionsSpy, + loggerMock + ); }); describe('path', () => { it('should return the path serializing buffer', () => { // arrange const expected = '/path/file'; - when(buffer.path).thenReturn(Buffer.from(expected)); + when(streamMock.path).thenReturn(Buffer.from(expected)); // act const result = harExporter.path; @@ -51,7 +70,7 @@ describe('DefaultHarExporter', () => { it('should return the path', () => { // arrange const expected = '/path/file'; - when(buffer.path).thenReturn(expected); + when(streamMock.path).thenReturn(expected); // act const result = harExporter.path; @@ -67,7 +86,7 @@ describe('DefaultHarExporter', () => { harExporter.end(); // assert - verify(buffer.end()).once(); + verify(streamMock.end()).once(); }); }); @@ -75,20 +94,22 @@ describe('DefaultHarExporter', () => { it('should write the entry to the buffer', async () => { // arrange // @ts-expect-error type mismatch - when(buffer.closed).thenReturn(false); + when(streamMock.closed).thenReturn(false); + when(optionsSpy.predicate).thenReturn(predicate); predicate.mockReturnValue(false); // act await harExporter.write(networkRequest); // assert - verify(buffer.write(match(`${EOL}`))).once(); + verify(streamMock.write(match(`${EOL}`))).once(); }); it('should write the entry to the buffer if the predicate returns throws an error', async () => { // arrange // @ts-expect-error type mismatch - when(buffer.closed).thenReturn(false); + when(streamMock.closed).thenReturn(false); + when(optionsSpy.predicate).thenReturn(predicate); predicate.mockReturnValue( Promise.reject(new Error('something went wrong')) ); @@ -97,33 +118,67 @@ describe('DefaultHarExporter', () => { await harExporter.write(networkRequest); // assert - verify(buffer.write(match(`${EOL}`))).once(); + verify(streamMock.write(match(`${EOL}`))).once(); + }); + + it('should transform the entry before writing to the buffer', async () => { + // arrange + const entry = { foo: 'bar' } as unknown as Entry; + const entryString = JSON.stringify(entry); + // @ts-expect-error type mismatch + when(streamMock.closed).thenReturn(false); + when(optionsSpy.transform).thenReturn(transform); + transform.mockReturnValue(Promise.resolve(entry)); + + // act + await harExporter.write(networkRequest); + + // assert + verify(streamMock.write(match(`${entryString}${EOL}`))).once(); + }); + + it('should skip the entry when the transformation is failed with an error', async () => { + // arrange + // @ts-expect-error type mismatch + when(streamMock.closed).thenReturn(false); + when(optionsSpy.transform).thenReturn(transform); + transform.mockReturnValue( + Promise.reject(new Error('Something went wrong.')) + ); + + // act + await harExporter.write(networkRequest); + + // assert + verify(streamMock.write(anyString())).never(); }); it('should not write the entry to the buffer if the predicate returns true', async () => { // arrange // @ts-expect-error type mismatch - when(buffer.closed).thenReturn(false); + when(streamMock.closed).thenReturn(false); + when(optionsSpy.predicate).thenReturn(predicate); predicate.mockReturnValue(true); // act await harExporter.write(networkRequest); // assert - verify(buffer.write(anyString())).never(); + verify(streamMock.write(anyString())).never(); }); it('should not write the entry to the buffer if the buffer is closed', async () => { // arrange // @ts-expect-error type mismatch - when(buffer.closed).thenReturn(true); + when(streamMock.closed).thenReturn(true); + when(optionsSpy.predicate).thenReturn(predicate); predicate.mockReturnValue(false); // act await harExporter.write(networkRequest); // assert - verify(buffer.write(anyString())).never(); + verify(streamMock.write(anyString())).never(); }); }); }); diff --git a/src/network/DefaultHarExporter.ts b/src/network/DefaultHarExporter.ts index 7293977..788724f 100644 --- a/src/network/DefaultHarExporter.ts +++ b/src/network/DefaultHarExporter.ts @@ -1,9 +1,17 @@ import { EntryBuilder } from './EntryBuilder'; import type { NetworkRequest } from './NetworkRequest'; import type { HarExporter } from './HarExporter'; +import type { Logger } from '../utils/Logger'; +import { ErrorUtils } from '../utils/ErrorUtils'; +import type { + DefaultHarExporterOptions, + Predicate, + Transformer +} from './DefaultHarExporterOptions'; import type { Entry } from 'har-format'; import type { WriteStream } from 'fs'; import { EOL } from 'os'; +import { format } from 'util'; export class DefaultHarExporter implements HarExporter { get path(): string { @@ -12,9 +20,18 @@ export class DefaultHarExporter implements HarExporter { return Buffer.isBuffer(path) ? path.toString('utf-8') : path; } + private get predicate(): Predicate | undefined { + return this.options?.predicate; + } + + private get transform(): Transformer | undefined { + return this.options?.transform; + } + constructor( + private readonly logger: Logger, private readonly buffer: WriteStream, - private readonly predicate?: (entry: Entry) => Promise | unknown + private readonly options?: DefaultHarExporterOptions ) {} public async write(networkRequest: NetworkRequest): Promise { @@ -24,24 +41,56 @@ export class DefaultHarExporter implements HarExporter { return; } - const json = JSON.stringify(entry); + const json = await this.serializeEntry(entry); // @ts-expect-error type mismatch - if (!this.buffer.closed) { + if (!this.buffer.closed && json) { this.buffer.write(`${json}${EOL}`); } } + public async serializeEntry(entry: Entry): Promise { + try { + const result = + typeof this.transform === 'function' + ? await this.transform(entry) + : entry; + + return JSON.stringify(result); + } catch (e) { + const stack = ErrorUtils.isError(e) ? e.stack : e; + const formattedEntry = format('%j', entry); + + this.logger.err( + `The entry is missing as a result of an error in the 'transform' function. + +The passed entry: +${formattedEntry} + +The stack trace for this error is: +${stack}` + ); + + return undefined; + } + } + public end(): void { this.buffer.end(); } - private async applyPredicate(entry: Entry) { + private async applyPredicate(entry: Entry): Promise { try { return ( typeof this.predicate === 'function' && (await this.predicate?.(entry)) ); - } catch { + } catch (e) { + const message = ErrorUtils.isError(e) ? e.message : e; + + this.logger.debug( + `The operation has encountered an error while processing the entry. ${message}` + ); + return false; } } diff --git a/src/network/DefaultHarExporterFactory.spec.ts b/src/network/DefaultHarExporterFactory.spec.ts index 40963e1..41d90c7 100644 --- a/src/network/DefaultHarExporterFactory.spec.ts +++ b/src/network/DefaultHarExporterFactory.spec.ts @@ -2,6 +2,7 @@ import type { FileManager } from '../utils/FileManager'; import { DefaultHarExporterFactory } from './DefaultHarExporterFactory'; import { DefaultHarExporter } from './DefaultHarExporter'; import { Loader } from '../utils/Loader'; +import type { Logger } from '../utils/Logger'; import { instance, mock, reset, spy, verify, when } from 'ts-mockito'; import { jest, @@ -31,31 +32,44 @@ const resolvableInstance = (m: T): T => describe('DefaultHarExporterFactory', () => { const fileManagerMock = mock(); const writeStreamMock = mock(); + const loggerMock = mock(); const loaderSpy = spy(Loader); const predicate = jest.fn(); + const transform = jest.fn(); let factory!: DefaultHarExporterFactory; beforeEach(() => { - factory = new DefaultHarExporterFactory(instance(fileManagerMock)); + factory = new DefaultHarExporterFactory( + instance(fileManagerMock), + instance(loggerMock) + ); }); - afterEach(() => - reset( + afterEach(() => { + transform.mockReset(); + predicate.mockReset(); + reset( fileManagerMock, writeStreamMock, - loaderSpy - ) - ); + loaderSpy, + loggerMock + ); + }); describe('create', () => { it('should create a HarExporter instance', async () => { // arrange - const options = { rootDir: '/root', predicatePath: 'predicate.js' }; + const options = { + rootDir: '/root', + predicatePath: 'predicate.js', + transformPath: 'transform.js' + }; when(fileManagerMock.createTmpWriteStream()).thenResolve( resolvableInstance(writeStreamMock) ); when(loaderSpy.load('/root/predicate.js')).thenResolve(predicate); + when(loaderSpy.load('/root/transform.js')).thenResolve(transform); // act const result = await factory.create(options); @@ -65,7 +79,7 @@ describe('DefaultHarExporterFactory', () => { verify(loaderSpy.load('/root/predicate.js')).once(); }); - it('should create a HarExporter instance without predicate', async () => { + it('should create a HarExporter instance without pre and post processors', async () => { // arrange const options = { rootDir: '/root' }; when(fileManagerMock.createTmpWriteStream()).thenResolve( diff --git a/src/network/DefaultHarExporterFactory.ts b/src/network/DefaultHarExporterFactory.ts index 6f219aa..f453f07 100644 --- a/src/network/DefaultHarExporterFactory.ts +++ b/src/network/DefaultHarExporterFactory.ts @@ -5,27 +5,54 @@ import type { import type { HarExporter } from './HarExporter'; import { DefaultHarExporter } from './DefaultHarExporter'; import { Loader } from '../utils/Loader'; -import { FileManager } from '../utils/FileManager'; -import type { Entry } from 'har-format'; +import type { FileManager } from '../utils/FileManager'; +import type { Logger } from '../utils/Logger'; +import type { + DefaultHarExporterOptions, + Predicate, + Transformer +} from './DefaultHarExporterOptions'; import { resolve } from 'path'; export class DefaultHarExporterFactory implements HarExporterFactory { - constructor(private readonly fileManager: FileManager) {} + constructor( + private readonly fileManager: FileManager, + private readonly logger: Logger + ) {} + + public async create(options: HarExporterOptions): Promise { + const settings = await this.createSettings(options); + const stream = await this.fileManager.createTmpWriteStream(); + + return new DefaultHarExporter(this.logger, stream, settings); + } + + private async createSettings({ + predicatePath, + transformPath, + rootDir + }: HarExporterOptions) { + const [predicate, transform]: (Predicate | Transformer | undefined)[] = + await Promise.all( + [predicatePath, transformPath].map(path => + this.loadCustomProcessor(path, rootDir) + ) + ); + + return { predicate, transform } as DefaultHarExporterOptions; + } - public async create({ - rootDir, - predicatePath - }: HarExporterOptions): Promise { - let predicate: ((request: Entry) => unknown) | undefined; + private async loadCustomProcessor( + path: string | undefined, + rootDir: string + ): Promise { + let processor: T | undefined; - if (predicatePath) { - const absolutePath = resolve(rootDir, predicatePath); - predicate = await Loader.load(absolutePath); + if (path) { + const absolutePath = resolve(rootDir, path); + processor = await Loader.load(absolutePath); } - return new DefaultHarExporter( - await this.fileManager.createTmpWriteStream(), - predicate - ); + return processor; } } diff --git a/src/network/DefaultHarExporterOptions.ts b/src/network/DefaultHarExporterOptions.ts new file mode 100644 index 0000000..3aceacf --- /dev/null +++ b/src/network/DefaultHarExporterOptions.ts @@ -0,0 +1,9 @@ +import type { Entry } from 'har-format'; + +export type Predicate = (entry: Entry) => Promise | unknown; +export type Transformer = (entry: Entry) => Promise | Entry; + +export interface DefaultHarExporterOptions { + predicate?: Predicate; + transform?: Transformer; +} diff --git a/src/network/HarExporterFactory.ts b/src/network/HarExporterFactory.ts index 168080b..73194db 100644 --- a/src/network/HarExporterFactory.ts +++ b/src/network/HarExporterFactory.ts @@ -3,6 +3,7 @@ import type { HarExporter } from './HarExporter'; export interface HarExporterOptions { rootDir: string; predicatePath?: string; + transformPath?: string; } export interface HarExporterFactory { diff --git a/src/network/index.ts b/src/network/index.ts index a78e53f..75bc308 100644 --- a/src/network/index.ts +++ b/src/network/index.ts @@ -21,3 +21,4 @@ export { HarBuilder } from './HarBuilder'; export { NetworkIdleMonitor } from './NetworkIdleMonitor'; export { NetworkObserver } from './NetworkObserver'; export { NetworkRequest, WebSocketFrameType } from './NetworkRequest'; +export type { DefaultHarExporterOptions } from './DefaultHarExporterOptions'; From 58bf1bae6d5ac81c2e01148a7199a0ce78e54d18 Mon Sep 17 00:00:00 2001 From: Artem Derevnjuk Date: Wed, 1 Feb 2023 16:42:39 +0400 Subject: [PATCH 2/9] docs(readme): add a description for the transform option closes #174 --- README.md | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 213ba58..e7c4e2f 100644 --- a/README.md +++ b/README.md @@ -242,10 +242,44 @@ export default async (entry: Entry) => { }; ``` -> ✴ The plugin also supports files with `.js`, `.mjs` and `.cjs` extensions. - In this example, the `filter` function will only exclude entries in the HAR where the request body contains a JSON object with a password field. +You can also modify the entries before saving the HAR by using the transform option. This option should be set to a path to a module that exports a function, similar to the filter option, to change the Entry object as desired. +The function should take an Entry object as a parameter and return the modified Entry object. + +```js +cy.recordHar({ transform: '../support/remove-sensitive-data.ts' }); +``` + +Here's a simple example of what the `remove-sensitive-data.ts` module might look like: + +```ts +import { Entry } from 'har-format'; + +const PASSWORD_REGEXP = /\"password":.*?(?=,)/g; + +export default async (entry: Entry) => { + try { + // Remove sensitive information from the request and response bodies + entry.request.postData.text = entry.request.postData.text.replace( + PASSWORD_REGEXP, + `"password": "***"` + ); + entry.response.content.text = entry.response.content.text.replace( + PASSWORD_REGEXP, + `"password": "***"` + ); + return entry; + } catch { + return entry; + } +}; +``` + +In this example, the transform function will replace any instances of password with `***` in both the request and response bodies of each entry. + +> ✴ The plugin also supports files with `.js`, `.mjs` and `.cjs` extensions. + By default, the path is relative to the spec folder. But by providing a `rootDir` it will look for the module in the provided directory: ```js From b8f96f170bb891bd24fe125bf90c0d7d12d2ee19 Mon Sep 17 00:00:00 2001 From: Artem Derevnjuk Date: Wed, 1 Feb 2023 17:32:03 +0400 Subject: [PATCH 3/9] refactor(plugin): unify option names through layers closes #174 --- src/Plugin.ts | 13 +++++++------ src/network/DefaultHarExporter.spec.ts | 8 ++++---- src/network/DefaultHarExporter.ts | 14 ++++++-------- src/network/DefaultHarExporterFactory.ts | 19 ++++++++++--------- src/network/DefaultHarExporterOptions.ts | 4 ++-- src/network/HarExporterFactory.ts | 4 ++-- 6 files changed, 31 insertions(+), 31 deletions(-) diff --git a/src/Plugin.ts b/src/Plugin.ts index 28857eb..375bec3 100644 --- a/src/Plugin.ts +++ b/src/Plugin.ts @@ -7,7 +7,12 @@ import type { Observer, ObserverFactory } from './network'; -import { HarBuilder, NetworkIdleMonitor, NetworkRequest } from './network'; +import { + HarBuilder, + HarExporterOptions, + NetworkIdleMonitor, + NetworkRequest +} from './network'; import { ErrorUtils } from './utils/ErrorUtils'; import type { Connection, ConnectionFactory } from './cdp'; import { @@ -25,11 +30,7 @@ export interface SaveOptions { waitForIdle?: boolean; } -export type RecordOptions = NetworkObserverOptions & { - rootDir: string; - filter?: string; - transform?: string; -}; +export type RecordOptions = NetworkObserverOptions & HarExporterOptions; interface Addr { port: number; diff --git a/src/network/DefaultHarExporter.spec.ts b/src/network/DefaultHarExporter.spec.ts index cc9fe2a..66fc6f3 100644 --- a/src/network/DefaultHarExporter.spec.ts +++ b/src/network/DefaultHarExporter.spec.ts @@ -95,7 +95,7 @@ describe('DefaultHarExporter', () => { // arrange // @ts-expect-error type mismatch when(streamMock.closed).thenReturn(false); - when(optionsSpy.predicate).thenReturn(predicate); + when(optionsSpy.filter).thenReturn(predicate); predicate.mockReturnValue(false); // act @@ -109,7 +109,7 @@ describe('DefaultHarExporter', () => { // arrange // @ts-expect-error type mismatch when(streamMock.closed).thenReturn(false); - when(optionsSpy.predicate).thenReturn(predicate); + when(optionsSpy.filter).thenReturn(predicate); predicate.mockReturnValue( Promise.reject(new Error('something went wrong')) ); @@ -157,7 +157,7 @@ describe('DefaultHarExporter', () => { // arrange // @ts-expect-error type mismatch when(streamMock.closed).thenReturn(false); - when(optionsSpy.predicate).thenReturn(predicate); + when(optionsSpy.filter).thenReturn(predicate); predicate.mockReturnValue(true); // act @@ -171,7 +171,7 @@ describe('DefaultHarExporter', () => { // arrange // @ts-expect-error type mismatch when(streamMock.closed).thenReturn(true); - when(optionsSpy.predicate).thenReturn(predicate); + when(optionsSpy.filter).thenReturn(predicate); predicate.mockReturnValue(false); // act diff --git a/src/network/DefaultHarExporter.ts b/src/network/DefaultHarExporter.ts index 788724f..e38fe03 100644 --- a/src/network/DefaultHarExporter.ts +++ b/src/network/DefaultHarExporter.ts @@ -5,7 +5,7 @@ import type { Logger } from '../utils/Logger'; import { ErrorUtils } from '../utils/ErrorUtils'; import type { DefaultHarExporterOptions, - Predicate, + Filter, Transformer } from './DefaultHarExporterOptions'; import type { Entry } from 'har-format'; @@ -20,8 +20,8 @@ export class DefaultHarExporter implements HarExporter { return Buffer.isBuffer(path) ? path.toString('utf-8') : path; } - private get predicate(): Predicate | undefined { - return this.options?.predicate; + private get filter(): Filter | undefined { + return this.options?.filter; } private get transform(): Transformer | undefined { @@ -37,7 +37,7 @@ export class DefaultHarExporter implements HarExporter { public async write(networkRequest: NetworkRequest): Promise { const entry = await new EntryBuilder(networkRequest).build(); - if (await this.applyPredicate(entry)) { + if (await this.applyFilter(entry)) { return; } @@ -79,11 +79,9 @@ ${stack}` this.buffer.end(); } - private async applyPredicate(entry: Entry): Promise { + private async applyFilter(entry: Entry): Promise { try { - return ( - typeof this.predicate === 'function' && (await this.predicate?.(entry)) - ); + return typeof this.filter === 'function' && (await this.filter(entry)); } catch (e) { const message = ErrorUtils.isError(e) ? e.message : e; diff --git a/src/network/DefaultHarExporterFactory.ts b/src/network/DefaultHarExporterFactory.ts index f453f07..66aaefc 100644 --- a/src/network/DefaultHarExporterFactory.ts +++ b/src/network/DefaultHarExporterFactory.ts @@ -9,7 +9,7 @@ import type { FileManager } from '../utils/FileManager'; import type { Logger } from '../utils/Logger'; import type { DefaultHarExporterOptions, - Predicate, + Filter, Transformer } from './DefaultHarExporterOptions'; import { resolve } from 'path'; @@ -28,21 +28,22 @@ export class DefaultHarExporterFactory implements HarExporterFactory { } private async createSettings({ - predicatePath, - transformPath, + filter, + transform, rootDir }: HarExporterOptions) { - const [predicate, transform]: (Predicate | Transformer | undefined)[] = + const [preProcessor, postProcessor]: (Filter | Transformer | undefined)[] = await Promise.all( - [predicatePath, transformPath].map(path => - this.loadCustomProcessor(path, rootDir) - ) + [filter, transform].map(path => this.loadCustomProcessor(path, rootDir)) ); - return { predicate, transform } as DefaultHarExporterOptions; + return { + filter: preProcessor, + transform: postProcessor + } as DefaultHarExporterOptions; } - private async loadCustomProcessor( + private async loadCustomProcessor( path: string | undefined, rootDir: string ): Promise { diff --git a/src/network/DefaultHarExporterOptions.ts b/src/network/DefaultHarExporterOptions.ts index 3aceacf..1b8f7b2 100644 --- a/src/network/DefaultHarExporterOptions.ts +++ b/src/network/DefaultHarExporterOptions.ts @@ -1,9 +1,9 @@ import type { Entry } from 'har-format'; -export type Predicate = (entry: Entry) => Promise | unknown; +export type Filter = (entry: Entry) => Promise | unknown; export type Transformer = (entry: Entry) => Promise | Entry; export interface DefaultHarExporterOptions { - predicate?: Predicate; + filter?: Filter; transform?: Transformer; } diff --git a/src/network/HarExporterFactory.ts b/src/network/HarExporterFactory.ts index 73194db..cb65f24 100644 --- a/src/network/HarExporterFactory.ts +++ b/src/network/HarExporterFactory.ts @@ -2,8 +2,8 @@ import type { HarExporter } from './HarExporter'; export interface HarExporterOptions { rootDir: string; - predicatePath?: string; - transformPath?: string; + filter?: string; + transform?: string; } export interface HarExporterFactory { From 61b587e6817065799c63561186502a0abb185221 Mon Sep 17 00:00:00 2001 From: Artem Derevnjuk Date: Wed, 1 Feb 2023 17:32:45 +0400 Subject: [PATCH 4/9] docs(readme): improve the usage example of transform option closes #174 --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e7c4e2f..4ff12d3 100644 --- a/README.md +++ b/README.md @@ -256,18 +256,18 @@ Here's a simple example of what the `remove-sensitive-data.ts` module might look ```ts import { Entry } from 'har-format'; -const PASSWORD_REGEXP = /\"password":.*?(?=,)/g; +const PASSWORD_REGEXP = /"password":.*?(?=,)/g; export default async (entry: Entry) => { try { // Remove sensitive information from the request and response bodies entry.request.postData.text = entry.request.postData.text.replace( PASSWORD_REGEXP, - `"password": "***"` + `"password":"***"` ); entry.response.content.text = entry.response.content.text.replace( PASSWORD_REGEXP, - `"password": "***"` + `"password":"***"` ); return entry; } catch { From a1b2852defe7a247edacba193eb52f48f645b877 Mon Sep 17 00:00:00 2001 From: Artem Derevnjuk Date: Wed, 1 Feb 2023 17:33:07 +0400 Subject: [PATCH 5/9] test: add more e2e tests closes #174 --- cypress/e2e/record-har.cy.ts | 18 ++++++++++++++++++ cypress/fixtures/filter.cjs | 4 ++-- cypress/fixtures/filter.js | 4 ++-- cypress/fixtures/filter.mjs | 4 ++-- cypress/fixtures/filter.ts | 4 ++-- cypress/fixtures/transform.cjs | 19 +++++++++++++++++++ cypress/fixtures/transform.js | 17 +++++++++++++++++ cypress/fixtures/transform.mjs | 17 +++++++++++++++++ cypress/fixtures/transform.ts | 19 +++++++++++++++++++ 9 files changed, 98 insertions(+), 8 deletions(-) create mode 100644 cypress/fixtures/transform.cjs create mode 100644 cypress/fixtures/transform.js create mode 100644 cypress/fixtures/transform.mjs create mode 100644 cypress/fixtures/transform.ts diff --git a/cypress/e2e/record-har.cy.ts b/cypress/e2e/record-har.cy.ts index d2254b2..4138ce2 100644 --- a/cypress/e2e/record-har.cy.ts +++ b/cypress/e2e/record-har.cy.ts @@ -41,6 +41,24 @@ describe('Record HAR', () => { }) ); + ['.js', '.ts', '.cjs'].forEach(ext => + it(`transforms a request using the custom postprocessor from ${ext} file`, () => { + cy.recordHar({ transform: `../fixtures/transform${ext}` }); + + cy.get('a[href$=fetch]').click(); + + cy.saveHar({ waitForIdle: true }); + + cy.findHar() + .its('log.entries') + .should('contain.something.like', { + response: { + content: { text: /\{"items":\[/ } + } + }); + }) + ); + it('excludes a request by its path', () => { cy.recordHar({ excludePaths: [/^\/api\/products$/, '^\\/api\\/users$'] }); diff --git a/cypress/fixtures/filter.cjs b/cypress/fixtures/filter.cjs index e8574f0..1c782b8 100644 --- a/cypress/fixtures/filter.cjs +++ b/cypress/fixtures/filter.cjs @@ -1,6 +1,6 @@ -module.exports = async req => { +module.exports = async entry => { try { - return /\{"products":\[/.test(req.response.content.text ?? ''); + return /\{"products":\[/.test(entry.response.content.text ?? ''); } catch { return false; } diff --git a/cypress/fixtures/filter.js b/cypress/fixtures/filter.js index e8574f0..1c782b8 100644 --- a/cypress/fixtures/filter.js +++ b/cypress/fixtures/filter.js @@ -1,6 +1,6 @@ -module.exports = async req => { +module.exports = async entry => { try { - return /\{"products":\[/.test(req.response.content.text ?? ''); + return /\{"products":\[/.test(entry.response.content.text ?? ''); } catch { return false; } diff --git a/cypress/fixtures/filter.mjs b/cypress/fixtures/filter.mjs index 6cd5883..5184bb3 100644 --- a/cypress/fixtures/filter.mjs +++ b/cypress/fixtures/filter.mjs @@ -1,6 +1,6 @@ -export default async req => { +export default async entry => { try { - return /\{"products":\[/.test(req.response.content.text ?? ''); + return /\{"products":\[/.test(entry.response.content.text ?? ''); } catch { return false; } diff --git a/cypress/fixtures/filter.ts b/cypress/fixtures/filter.ts index 9f69574..52a500d 100644 --- a/cypress/fixtures/filter.ts +++ b/cypress/fixtures/filter.ts @@ -1,8 +1,8 @@ import { Entry } from 'har-format'; -export default async (req: Entry) => { +export default async (entry: Entry) => { try { - return /\{"products":\[/.test(req.response.content.text ?? ''); + return /\{"products":\[/.test(entry.response.content.text ?? ''); } catch { return false; } diff --git a/cypress/fixtures/transform.cjs b/cypress/fixtures/transform.cjs new file mode 100644 index 0000000..2b26c15 --- /dev/null +++ b/cypress/fixtures/transform.cjs @@ -0,0 +1,19 @@ +module.exports = async entry => { + try { + if (entry.response.content?.text) { + entry.response.content = { + ...entry.response.content, + text: entry.response.content.text.replace( + /"products":\s*(.+)/g, + `"items":$1` + ) + }; + } + + return entry; + } catch { + return entry; + } +}; + + diff --git a/cypress/fixtures/transform.js b/cypress/fixtures/transform.js new file mode 100644 index 0000000..d120b62 --- /dev/null +++ b/cypress/fixtures/transform.js @@ -0,0 +1,17 @@ +module.exports = async entry => { + try { + if (entry.response.content?.text) { + entry.response.content = { + ...entry.response.content, + text: entry.response.content.text.replace( + /"products":\s*(.+)/g, + `"items":$1` + ) + }; + } + + return entry; + } catch { + return entry; + } +}; diff --git a/cypress/fixtures/transform.mjs b/cypress/fixtures/transform.mjs new file mode 100644 index 0000000..705b031 --- /dev/null +++ b/cypress/fixtures/transform.mjs @@ -0,0 +1,17 @@ +export async entry => { + try { + if (entry.response.content?.text) { + entry.response.content = { + ...entry.response.content, + text: entry.response.content.text.replace( + /"products":\s*(.+)/g, + `"items":$1` + ) + }; + } + + return entry; + } catch { + return entry; + } +}; diff --git a/cypress/fixtures/transform.ts b/cypress/fixtures/transform.ts new file mode 100644 index 0000000..5db4b5b --- /dev/null +++ b/cypress/fixtures/transform.ts @@ -0,0 +1,19 @@ +import { Entry } from 'har-format'; + +export default async (entry: Entry) => { + try { + if (entry.response.content?.text) { + entry.response.content = { + ...entry.response.content, + text: entry.response.content.text.replace( + /"products":\s*(.+)/g, + `"items":$1` + ) + }; + } + + return entry; + } catch { + return entry; + } +}; From e23de095dc8d181d122d1d951b76d322ec710cb5 Mon Sep 17 00:00:00 2001 From: Artem Derevnjuk Date: Wed, 1 Feb 2023 17:36:10 +0400 Subject: [PATCH 6/9] refactor(plugin): use the typed import closes #174 --- src/Plugin.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/Plugin.ts b/src/Plugin.ts index 375bec3..171a062 100644 --- a/src/Plugin.ts +++ b/src/Plugin.ts @@ -5,14 +5,10 @@ import type { HarExporterFactory, NetworkObserverOptions, Observer, - ObserverFactory -} from './network'; -import { - HarBuilder, - HarExporterOptions, - NetworkIdleMonitor, - NetworkRequest + ObserverFactory, + HarExporterOptions } from './network'; +import { HarBuilder, NetworkIdleMonitor, NetworkRequest } from './network'; import { ErrorUtils } from './utils/ErrorUtils'; import type { Connection, ConnectionFactory } from './cdp'; import { From b126debb8199815a966431f82ae9803c19ddbe41 Mon Sep 17 00:00:00 2001 From: Artem Derevnjuk Date: Wed, 1 Feb 2023 17:41:35 +0400 Subject: [PATCH 7/9] test(network): clarify the test data closes #174 --- src/network/DefaultHarExporterFactory.spec.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/network/DefaultHarExporterFactory.spec.ts b/src/network/DefaultHarExporterFactory.spec.ts index 41d90c7..87d3e5d 100644 --- a/src/network/DefaultHarExporterFactory.spec.ts +++ b/src/network/DefaultHarExporterFactory.spec.ts @@ -62,8 +62,8 @@ describe('DefaultHarExporterFactory', () => { // arrange const options = { rootDir: '/root', - predicatePath: 'predicate.js', - transformPath: 'transform.js' + filter: 'predicate.js', + transform: 'transform.js' }; when(fileManagerMock.createTmpWriteStream()).thenResolve( resolvableInstance(writeStreamMock) @@ -77,6 +77,7 @@ describe('DefaultHarExporterFactory', () => { // assert expect(result).toBeInstanceOf(DefaultHarExporter); verify(loaderSpy.load('/root/predicate.js')).once(); + verify(loaderSpy.load('/root/transform.js')).once(); }); it('should create a HarExporter instance without pre and post processors', async () => { From 099024a1bff7c59f195f25a9409bbe1175212e18 Mon Sep 17 00:00:00 2001 From: Artem Derevnjuk Date: Wed, 1 Feb 2023 17:55:07 +0400 Subject: [PATCH 8/9] docs(readme): reorder the statements closes #174 --- README.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 4ff12d3..4e26ec4 100644 --- a/README.md +++ b/README.md @@ -244,8 +244,7 @@ export default async (entry: Entry) => { In this example, the `filter` function will only exclude entries in the HAR where the request body contains a JSON object with a password field. -You can also modify the entries before saving the HAR by using the transform option. This option should be set to a path to a module that exports a function, similar to the filter option, to change the Entry object as desired. -The function should take an Entry object as a parameter and return the modified Entry object. +You can also modify the entries before saving the HAR by using the `transform` option. This option should be set to a path to a module that exports a function, similar to the filter option, to change the Entry object as desired. ```js cy.recordHar({ transform: '../support/remove-sensitive-data.ts' }); @@ -256,19 +255,18 @@ Here's a simple example of what the `remove-sensitive-data.ts` module might look ```ts import { Entry } from 'har-format'; -const PASSWORD_REGEXP = /"password":.*?(?=,)/g; +const PASSWORD_REGEXP = /("password":\s*")\w+("\s*)/; export default async (entry: Entry) => { try { - // Remove sensitive information from the request and response bodies - entry.request.postData.text = entry.request.postData.text.replace( - PASSWORD_REGEXP, - `"password":"***"` - ); - entry.response.content.text = entry.response.content.text.replace( - PASSWORD_REGEXP, - `"password":"***"` - ); + // Remove sensitive information from the request body + if (entry.request.postData?.text) { + entry.request.postData.text = entry.request.postData.text.replace( + PASSWORD_REGEXP, + '$1***$2' + ); + } + return entry; } catch { return entry; @@ -276,7 +274,9 @@ export default async (entry: Entry) => { }; ``` -In this example, the transform function will replace any instances of password with `***` in both the request and response bodies of each entry. +The function should take an Entry object as a parameter and return the modified. + +In this example, the transform function will replace any occurrences of password with `***` in the request body of each entry. > ✴ The plugin also supports files with `.js`, `.mjs` and `.cjs` extensions. From 31faebafc436b37ac9630c68d34e1c3302e8eda0 Mon Sep 17 00:00:00 2001 From: Artem Derevnjuk Date: Wed, 1 Feb 2023 18:33:56 +0400 Subject: [PATCH 9/9] refactor(network): improve the log messages closes #174 --- src/network/DefaultHarExporter.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/network/DefaultHarExporter.ts b/src/network/DefaultHarExporter.ts index e38fe03..2754580 100644 --- a/src/network/DefaultHarExporter.ts +++ b/src/network/DefaultHarExporter.ts @@ -59,14 +59,13 @@ export class DefaultHarExporter implements HarExporter { return JSON.stringify(result); } catch (e) { const stack = ErrorUtils.isError(e) ? e.stack : e; - const formattedEntry = format('%j', entry); + this.logger.debug( + format(`The entry has been filtered out due to an error: %j`, entry) + ); this.logger.err( `The entry is missing as a result of an error in the 'transform' function. - -The passed entry: -${formattedEntry} - +s The stack trace for this error is: ${stack}` );