From d588ada62d9fa91ca8eeb76b3a29aa68a3f2d824 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Thu, 2 Jan 2025 10:33:08 -0500 Subject: [PATCH 01/17] refactor(NODE-6616): short circuit EJSON stringifying --- src/mongo_logger.ts | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/src/mongo_logger.ts b/src/mongo_logger.ts index f83c16138ff..d49bbe6071a 100644 --- a/src/mongo_logger.ts +++ b/src/mongo_logger.ts @@ -1,4 +1,5 @@ import { inspect, promisify } from 'util'; +import { isUint8Array } from 'util/types'; import { type Document, EJSON, type EJSONOptions, type ObjectId } from './bson'; import type { CommandStartedEvent } from './cmap/command_monitoring_events'; @@ -421,13 +422,44 @@ export function stringifyWithMaxLen( ): string { let strToTruncate = ''; + let currentLength = 0; + const maxDocumentLengthEnsurer = function maxDocumentLengthEnsurer(key: string, value: any) { + if (currentLength >= maxDocumentLength) { + return undefined; + } + + currentLength += key.length + 4; + + if (typeof value === 'string') { + currentLength += value.length; + } else if (typeof value === 'number' || typeof value === 'bigint') { + currentLength += 20; + } else if (typeof value === 'boolean') { + currentLength += value ? 4 : 5; + } else if (value != null && typeof value === 'object' && '_bsontype' in value) { + if (isUint8Array(value.buffer)) { + currentLength += (value.buffer.byteLength + value.buffer.byteLength * 0.5) | 0; + } else if (value._bsontype === 'Binary') { + currentLength += (value.position + value.position * 0.3) | 0; + } else if (value._bsontype === 'Code') { + currentLength += value.code.length; + } + } + + return value; + }; + if (typeof value === 'string') { strToTruncate = value; } else if (typeof value === 'function') { strToTruncate = value.name; } else { try { - strToTruncate = EJSON.stringify(value, options); + if (maxDocumentLength !== 0) { + strToTruncate = EJSON.stringify(value, maxDocumentLengthEnsurer, 0, options); + } else { + strToTruncate = EJSON.stringify(value, options); + } } catch (e) { strToTruncate = `Extended JSON serialization failed with: ${e.message}`; } From a4cce5fef496eadec558d84a4d759f1defdbd556 Mon Sep 17 00:00:00 2001 From: Durran Jordan Date: Fri, 17 Jan 2025 16:16:02 +0100 Subject: [PATCH 02/17] chore(NODE-6637): remove drivers tools env setting (#4363) --- .evergreen/config.yml | 54 -------------------------- .evergreen/generate_evergreen_tasks.js | 2 +- .evergreen/install-dependencies.sh | 14 +------ 3 files changed, 3 insertions(+), 67 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index a3dbad1e7a3..828e804f6f9 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1383,7 +1383,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: latest} - {key: TOPOLOGY, value: server} - {key: AUTH, value: auth} @@ -1400,7 +1399,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: latest} - {key: TOPOLOGY, value: replica_set} - {key: AUTH, value: auth} @@ -1417,7 +1415,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: latest} - {key: TOPOLOGY, value: sharded_cluster} - {key: AUTH, value: auth} @@ -1434,7 +1431,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: rapid} - {key: TOPOLOGY, value: server} - {key: AUTH, value: auth} @@ -1451,7 +1447,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: rapid} - {key: TOPOLOGY, value: replica_set} - {key: AUTH, value: auth} @@ -1468,7 +1463,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: rapid} - {key: TOPOLOGY, value: sharded_cluster} - {key: AUTH, value: auth} @@ -1485,7 +1479,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '8.0'} - {key: TOPOLOGY, value: server} - {key: AUTH, value: auth} @@ -1502,7 +1495,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '8.0'} - {key: TOPOLOGY, value: replica_set} - {key: AUTH, value: auth} @@ -1519,7 +1511,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '8.0'} - {key: TOPOLOGY, value: sharded_cluster} - {key: AUTH, value: auth} @@ -1536,7 +1527,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '7.0'} - {key: TOPOLOGY, value: server} - {key: AUTH, value: auth} @@ -1553,7 +1543,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '7.0'} - {key: TOPOLOGY, value: replica_set} - {key: AUTH, value: auth} @@ -1570,7 +1559,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '7.0'} - {key: TOPOLOGY, value: sharded_cluster} - {key: AUTH, value: auth} @@ -1587,7 +1575,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '6.0'} - {key: TOPOLOGY, value: server} - {key: AUTH, value: auth} @@ -1604,7 +1591,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '6.0'} - {key: TOPOLOGY, value: replica_set} - {key: AUTH, value: auth} @@ -1621,7 +1607,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '6.0'} - {key: TOPOLOGY, value: sharded_cluster} - {key: AUTH, value: auth} @@ -1638,7 +1623,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '5.0'} - {key: TOPOLOGY, value: server} - {key: AUTH, value: auth} @@ -1655,7 +1639,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '5.0'} - {key: TOPOLOGY, value: replica_set} - {key: AUTH, value: auth} @@ -1672,7 +1655,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '5.0'} - {key: TOPOLOGY, value: sharded_cluster} - {key: AUTH, value: auth} @@ -1689,7 +1671,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '4.4'} - {key: TOPOLOGY, value: server} - {key: AUTH, value: auth} @@ -1706,7 +1687,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '4.4'} - {key: TOPOLOGY, value: replica_set} - {key: AUTH, value: auth} @@ -1723,7 +1703,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '4.4'} - {key: TOPOLOGY, value: sharded_cluster} - {key: AUTH, value: auth} @@ -1740,7 +1719,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '4.2'} - {key: TOPOLOGY, value: server} - {key: AUTH, value: auth} @@ -1757,7 +1735,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '4.2'} - {key: TOPOLOGY, value: replica_set} - {key: AUTH, value: auth} @@ -1774,7 +1751,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '4.2'} - {key: TOPOLOGY, value: sharded_cluster} - {key: AUTH, value: auth} @@ -1791,7 +1767,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '4.0'} - {key: TOPOLOGY, value: server} - {key: AUTH, value: auth} @@ -1808,7 +1783,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '4.0'} - {key: TOPOLOGY, value: replica_set} - {key: AUTH, value: auth} @@ -1825,7 +1799,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '4.0'} - {key: TOPOLOGY, value: sharded_cluster} - {key: AUTH, value: auth} @@ -3860,7 +3833,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: latest} - {key: TOPOLOGY, value: server} - {key: AUTH, value: noauth} @@ -3878,7 +3850,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: latest} - {key: TOPOLOGY, value: replica_set} - {key: AUTH, value: noauth} @@ -3896,7 +3867,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: latest} - {key: TOPOLOGY, value: sharded_cluster} - {key: AUTH, value: noauth} @@ -3914,7 +3884,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: rapid} - {key: TOPOLOGY, value: server} - {key: AUTH, value: noauth} @@ -3932,7 +3901,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: rapid} - {key: TOPOLOGY, value: replica_set} - {key: AUTH, value: noauth} @@ -3950,7 +3918,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: rapid} - {key: TOPOLOGY, value: sharded_cluster} - {key: AUTH, value: noauth} @@ -3968,7 +3935,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '8.0'} - {key: TOPOLOGY, value: server} - {key: AUTH, value: noauth} @@ -3986,7 +3952,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '8.0'} - {key: TOPOLOGY, value: replica_set} - {key: AUTH, value: noauth} @@ -4004,7 +3969,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '8.0'} - {key: TOPOLOGY, value: sharded_cluster} - {key: AUTH, value: noauth} @@ -4022,7 +3986,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '7.0'} - {key: TOPOLOGY, value: server} - {key: AUTH, value: noauth} @@ -4040,7 +4003,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '7.0'} - {key: TOPOLOGY, value: replica_set} - {key: AUTH, value: noauth} @@ -4058,7 +4020,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '7.0'} - {key: TOPOLOGY, value: sharded_cluster} - {key: AUTH, value: noauth} @@ -4076,7 +4037,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '6.0'} - {key: TOPOLOGY, value: server} - {key: AUTH, value: noauth} @@ -4094,7 +4054,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '6.0'} - {key: TOPOLOGY, value: replica_set} - {key: AUTH, value: noauth} @@ -4112,7 +4071,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '6.0'} - {key: TOPOLOGY, value: sharded_cluster} - {key: AUTH, value: noauth} @@ -4130,7 +4088,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '5.0'} - {key: TOPOLOGY, value: server} - {key: AUTH, value: noauth} @@ -4148,7 +4105,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '5.0'} - {key: TOPOLOGY, value: replica_set} - {key: AUTH, value: noauth} @@ -4166,7 +4122,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '5.0'} - {key: TOPOLOGY, value: sharded_cluster} - {key: AUTH, value: noauth} @@ -4184,7 +4139,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '4.4'} - {key: TOPOLOGY, value: server} - {key: AUTH, value: noauth} @@ -4202,7 +4156,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '4.4'} - {key: TOPOLOGY, value: replica_set} - {key: AUTH, value: noauth} @@ -4220,7 +4173,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '4.4'} - {key: TOPOLOGY, value: sharded_cluster} - {key: AUTH, value: noauth} @@ -4238,7 +4190,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '4.2'} - {key: TOPOLOGY, value: server} - {key: AUTH, value: noauth} @@ -4256,7 +4207,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '4.2'} - {key: TOPOLOGY, value: replica_set} - {key: AUTH, value: noauth} @@ -4274,7 +4224,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '4.2'} - {key: TOPOLOGY, value: sharded_cluster} - {key: AUTH, value: noauth} @@ -4292,7 +4241,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '4.0'} - {key: TOPOLOGY, value: server} - {key: AUTH, value: noauth} @@ -4310,7 +4258,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '4.0'} - {key: TOPOLOGY, value: replica_set} - {key: AUTH, value: noauth} @@ -4328,7 +4275,6 @@ tasks: type: setup params: updates: - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '4.0'} - {key: TOPOLOGY, value: sharded_cluster} - {key: AUTH, value: noauth} diff --git a/.evergreen/generate_evergreen_tasks.js b/.evergreen/generate_evergreen_tasks.js index ca11d5ff61e..516e84644a0 100644 --- a/.evergreen/generate_evergreen_tasks.js +++ b/.evergreen/generate_evergreen_tasks.js @@ -70,7 +70,7 @@ function makeTask({ mongoVersion, topology, tags = [], auth = 'auth' }) { name: `test-${mongoVersion}-${topology}${auth === 'noauth' ? '-noauth' : ''}`, tags: [mongoVersion, topology, ...tags], commands: [ - updateExpansions({ NPM_VERSION: 9, VERSION: mongoVersion, TOPOLOGY: topology, AUTH: auth }), + updateExpansions({ VERSION: mongoVersion, TOPOLOGY: topology, AUTH: auth }), { func: 'install dependencies' }, { func: 'bootstrap mongo-orchestration' }, { func: 'bootstrap kms servers' }, diff --git a/.evergreen/install-dependencies.sh b/.evergreen/install-dependencies.sh index 99b4d376d4c..33b9958b885 100644 --- a/.evergreen/install-dependencies.sh +++ b/.evergreen/install-dependencies.sh @@ -7,18 +7,8 @@ set -o errexit # Exit the script with error if any of the commands fail ## a full nodejs version, in the format v..patch export NODE_LTS_VERSION=${NODE_LTS_VERSION:-16} # npm version can be defined in the environment for cases where we need to install -# a version lower than latest to support EOL Node versions. - -# If NODE_LTS_VERSION is numeric and less than 18, default to 9, if less than 20, default to 10. -# Do not override if it is already set. -if [[ "$NODE_LTS_VERSION" =~ ^[0-9]+$ && "$NODE_LTS_VERSION" -lt 18 ]]; then - export NPM_VERSION=${NPM_VERSION:-9} -elif [[ "$NODE_LTS_VERSION" =~ ^[0-9]+$ && "$NODE_LTS_VERSION" -lt 20 ]]; then - export NPM_VERSION=${NPM_VERSION:-10} -else - export NPM_VERSION=${NPM_VERSION:-latest} -fi - +# a version lower than latest to support EOL Node versions. When not provided will +# be handled by this script in drivers tools. source $DRIVERS_TOOLS/.evergreen/install-node.sh npm install "${NPM_OPTIONS}" From d21c4ffb1b43835d0936c1a506e8f39f61dd226c Mon Sep 17 00:00:00 2001 From: Warren James Date: Fri, 17 Jan 2025 18:16:50 -0500 Subject: [PATCH 03/17] perf(NODE-6452): Optimize CommandStartedEvent and CommandSucceededEvent constructors (#4371) --- src/cmap/command_monitoring_events.ts | 65 +++---------------- src/utils.ts | 37 ----------- .../command_monitoring.test.ts | 45 ------------- 3 files changed, 10 insertions(+), 137 deletions(-) diff --git a/src/cmap/command_monitoring_events.ts b/src/cmap/command_monitoring_events.ts index a7e1653becc..992b9dc47c9 100644 --- a/src/cmap/command_monitoring_events.ts +++ b/src/cmap/command_monitoring_events.ts @@ -6,7 +6,7 @@ import { LEGACY_HELLO_COMMAND, LEGACY_HELLO_COMMAND_CAMEL_CASE } from '../constants'; -import { calculateDurationInMs, deepCopy } from '../utils'; +import { calculateDurationInMs } from '../utils'; import { DocumentSequence, OpMsgRequest, @@ -125,7 +125,7 @@ export class CommandSucceededEvent { this.requestId = command.requestId; this.commandName = commandName; this.duration = calculateDurationInMs(started); - this.reply = maybeRedact(commandName, cmd, extractReply(command, reply)); + this.reply = maybeRedact(commandName, cmd, extractReply(reply)); this.serverConnectionId = serverConnectionId; } @@ -214,7 +214,6 @@ const HELLO_COMMANDS = new Set(['hello', LEGACY_HELLO_COMMAND, LEGACY_HELLO_COMM // helper methods const extractCommandName = (commandDoc: Document) => Object.keys(commandDoc)[0]; -const namespace = (command: OpQueryRequest) => command.ns; const collectionName = (command: OpQueryRequest) => command.ns.split('.')[1]; const maybeRedact = (commandName: string, commandDoc: Document, result: Error | Document) => SENSITIVE_COMMANDS.has(commandName) || @@ -242,19 +241,10 @@ const LEGACY_FIND_OPTIONS_MAP = { returnFieldSelector: 'projection' } as const; -const OP_QUERY_KEYS = [ - 'tailable', - 'oplogReplay', - 'noCursorTimeout', - 'awaitData', - 'partial', - 'exhaust' -] as const; - /** Extract the actual command from the query, possibly up-converting if it's a legacy format */ function extractCommand(command: WriteProtocolMessageType): Document { if (command instanceof OpMsgRequest) { - const cmd = deepCopy(command.command); + const cmd = { ...command.command }; // For OP_MSG with payload type 1 we need to pull the documents // array out of the document sequence for monitoring. if (cmd.ops instanceof DocumentSequence) { @@ -276,7 +266,7 @@ function extractCommand(command: WriteProtocolMessageType): Document { result = { find: collectionName(command) }; Object.keys(LEGACY_FIND_QUERY_MAP).forEach(key => { if (command.query[key] != null) { - result[LEGACY_FIND_QUERY_MAP[key]] = deepCopy(command.query[key]); + result[LEGACY_FIND_QUERY_MAP[key]] = { ...command.query[key] }; } }); } @@ -284,64 +274,29 @@ function extractCommand(command: WriteProtocolMessageType): Document { Object.keys(LEGACY_FIND_OPTIONS_MAP).forEach(key => { const legacyKey = key as keyof typeof LEGACY_FIND_OPTIONS_MAP; if (command[legacyKey] != null) { - result[LEGACY_FIND_OPTIONS_MAP[legacyKey]] = deepCopy(command[legacyKey]); - } - }); - - OP_QUERY_KEYS.forEach(key => { - if (command[key]) { - result[key] = command[key]; + result[LEGACY_FIND_OPTIONS_MAP[legacyKey]] = command[legacyKey]; } }); - if (command.pre32Limit != null) { - result.limit = command.pre32Limit; - } - - if (command.query.$explain) { - return { explain: result }; - } return result; } - const clonedQuery: Record = {}; - const clonedCommand: Record = {}; + let clonedQuery: Record = {}; + const clonedCommand: Record = { ...command }; if (command.query) { - for (const k in command.query) { - clonedQuery[k] = deepCopy(command.query[k]); - } + clonedQuery = { ...command.query }; clonedCommand.query = clonedQuery; } - for (const k in command) { - if (k === 'query') continue; - clonedCommand[k] = deepCopy((command as unknown as Record)[k]); - } return command.query ? clonedQuery : clonedCommand; } -function extractReply(command: WriteProtocolMessageType, reply?: Document) { +function extractReply(reply?: Document) { if (!reply) { return reply; } - if (command instanceof OpMsgRequest) { - return deepCopy(reply.result ? reply.result : reply); - } - - // is this a legacy find command? - if (command.query && command.query.$query != null) { - return { - ok: 1, - cursor: { - id: deepCopy(reply.cursorId), - ns: namespace(command), - firstBatch: deepCopy(reply.documents) - } - }; - } - - return deepCopy(reply.result ? reply.result : reply); + return reply.result ? reply.result : reply; } function extractConnectionDetails(connection: Connection) { diff --git a/src/utils.ts b/src/utils.ts index c23161612a8..55cf455676b 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -620,43 +620,6 @@ export function isRecord( return true; } -/** - * Make a deep copy of an object - * - * NOTE: This is not meant to be the perfect implementation of a deep copy, - * but instead something that is good enough for the purposes of - * command monitoring. - */ -export function deepCopy(value: T): T { - if (value == null) { - return value; - } else if (Array.isArray(value)) { - return value.map(item => deepCopy(item)) as unknown as T; - } else if (isRecord(value)) { - const res = {} as any; - for (const key in value) { - res[key] = deepCopy(value[key]); - } - return res; - } - - const ctor = (value as any).constructor; - if (ctor) { - switch (ctor.name.toLowerCase()) { - case 'date': - return new ctor(Number(value)); - case 'map': - return new Map(value as any) as unknown as T; - case 'set': - return new Set(value as any) as unknown as T; - case 'buffer': - return Buffer.from(value as unknown as Buffer) as unknown as T; - } - } - - return value; -} - type ListNode = { value: T; next: ListNode | HeadNode; diff --git a/test/integration/command-logging-and-monitoring/command_monitoring.test.ts b/test/integration/command-logging-and-monitoring/command_monitoring.test.ts index 124d5e7f9a2..eb589346a0c 100644 --- a/test/integration/command-logging-and-monitoring/command_monitoring.test.ts +++ b/test/integration/command-logging-and-monitoring/command_monitoring.test.ts @@ -603,49 +603,4 @@ describe('Command Monitoring', function () { }); } }); - - describe('Internal state references', function () { - let client; - - beforeEach(function () { - client = this.configuration.newClient( - { writeConcern: { w: 1 } }, - { maxPoolSize: 1, monitorCommands: true } - ); - }); - - afterEach(function (done) { - client.close(done); - }); - - // NODE-1502 - it('should not allow mutation of internal state from commands returned by event monitoring', function () { - const started = []; - const succeeded = []; - client.on('commandStarted', filterForCommands('insert', started)); - client.on('commandSucceeded', filterForCommands('insert', succeeded)); - const documentToInsert = { a: { b: 1 } }; - const db = client.db(this.configuration.db); - return db - .collection('apm_test') - .insertOne(documentToInsert) - .then(r => { - expect(r).to.have.property('insertedId').that.is.an('object'); - expect(started).to.have.lengthOf(1); - // Check if contents of returned document are equal to document inserted (by value) - expect(documentToInsert).to.deep.equal(started[0].command.documents[0]); - // Check if the returned document is a clone of the original. This confirms that the - // reference is not the same. - expect(documentToInsert !== started[0].command.documents[0]).to.equal(true); - expect(documentToInsert.a !== started[0].command.documents[0].a).to.equal(true); - - started[0].command.documents[0].a.b = 2; - expect(documentToInsert.a.b).to.equal(1); - - expect(started[0].commandName).to.equal('insert'); - expect(started[0].command.insert).to.equal('apm_test'); - expect(succeeded).to.have.lengthOf(1); - }); - }); - }); }); From 31f941e4f7da4aec0cd614512ceb26c46c3f595e Mon Sep 17 00:00:00 2001 From: Warren James Date: Tue, 21 Jan 2025 15:58:14 -0500 Subject: [PATCH 04/17] finish accounting for other BSON types --- src/mongo_logger.ts | 125 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 116 insertions(+), 9 deletions(-) diff --git a/src/mongo_logger.ts b/src/mongo_logger.ts index d49bbe6071a..d880484be4a 100644 --- a/src/mongo_logger.ts +++ b/src/mongo_logger.ts @@ -1,7 +1,24 @@ import { inspect, promisify } from 'util'; import { isUint8Array } from 'util/types'; -import { type Document, EJSON, type EJSONOptions, type ObjectId } from './bson'; +import { + type Binary, + type BSONRegExp, + type BSONSymbol, + type Code, + type DBRef, + type Decimal128, + type Document, + type Double, + EJSON, + type EJSONOptions, + type Int32, + type Long, + type MaxKey, + type MinKey, + type ObjectId, + type Timestamp +} from './bson'; import type { CommandStartedEvent } from './cmap/command_monitoring_events'; import type { ConnectionCheckedInEvent, @@ -414,6 +431,20 @@ export interface LogConvertible extends Record { toLog(): Record; } +type BSONObject = + | BSONRegExp + | BSONSymbol + | Code + | DBRef + | Decimal128 + | Double + | Int32 + | Long + | MaxKey + | MinKey + | ObjectId + | Timestamp + | Binary; /** @internal */ export function stringifyWithMaxLen( value: any, @@ -427,22 +458,98 @@ export function stringifyWithMaxLen( if (currentLength >= maxDocumentLength) { return undefined; } + // Account for root document + if (key === '') { + // Account for starting brace + currentLength += 1; + return value; + } + // +4 accounts for 2 quotation marks, colon and comma after value currentLength += key.length + 4; if (typeof value === 'string') { - currentLength += value.length; + // +2 accounts for quotes + currentLength += value.length + 2; } else if (typeof value === 'number' || typeof value === 'bigint') { - currentLength += 20; + currentLength += String(value).length; } else if (typeof value === 'boolean') { currentLength += value ? 4 : 5; + } else if ('buffer' in value && isUint8Array(value.buffer)) { + // Handle binData + currentLength += (value.buffer.byteLength + value.buffer.byteLength * 0.5) | 0; } else if (value != null && typeof value === 'object' && '_bsontype' in value) { - if (isUint8Array(value.buffer)) { - currentLength += (value.buffer.byteLength + value.buffer.byteLength * 0.5) | 0; - } else if (value._bsontype === 'Binary') { - currentLength += (value.position + value.position * 0.3) | 0; - } else if (value._bsontype === 'Code') { - currentLength += value.code.length; + const v = value as BSONObject; + if (v._bsontype === 'Binary') { + // This is an estimate based on the fact that the base64 is approximately 1.3x the length of + // the actual binary sequence + // Also accounting for stringified fields before the binary sequence and the fields after + // the binary sequence + currentLength += (value.position + value.position * 0.3 + 22 + 17) | 0; + } else if (v._bsontype === 'Code') { + // '{"$code":""}' or '{"$code":"","$scope":}' + // TODO: Account for scope? + if (v.scope == null) { + currentLength += v.code.length + 10 + 2; + } else { + // Ignoring actual scope object + currentLength += v.code.length + 10 + 11; + } + } else if (v._bsontype === 'Decimal128') { + // TODO: Is this worth doing here? + currentLength += value.toExtendedJSON().length; + } else if (v._bsontype === 'Double') { + // Doesn't account for representing integers as .0 + if ('value' in v && typeof v.value === 'number') currentLength += String(v.value).length; + } else if (v._bsontype === 'Int32') { + if ('value' in v && typeof v.value === 'number') currentLength += String(v.value).length; + } else if (v._bsontype === 'Long') { + if ('toString' in v && typeof v.toString === 'function') { + currentLength += v.toString().length; + } + } else if (v._bsontype === 'MaxKey' || v._bsontype === 'MinKey') { + // '{"$maxKey":1}' or '{"$minKey":1}' + currentLength += 13; + } else if (v._bsontype === 'ObjectId') { + // '{"$oid":"XXXXXXXXXXXXXXXXXXXXXXXX"}' + currentLength += 35; + } else if ( + v._bsontype === 'BSONRegExp' && + 'pattern' in v && + typeof v.pattern === 'string' && + 'options' in v && + typeof v.options === 'string' + ) { + // '{"$regularExpression":{"pattern":"","options":""}}' + currentLength += 34 + v.pattern.length + 13 + v.options.length + 3; + } else if (v._bsontype === 'BSONSymbol' && 'value' in v && v.value === 'string') { + // '{"$symbol": ""}' + currentLength += 12 + v.value.length + 2; + } else if ( + v._bsontype === 'Timestamp' && + 't' in v && + typeof v.t === 'string' && + 'i' in v && + typeof v.i === 'string' + ) { + currentLength += 19 + String(v.t).length + 5 + String(v.i).length + 2; + } else if (v._bsontype === 'DBRef') { + // '{"$ref":"","$id":}' or '{"$ref":"","$id":,"$db":"test"}' + currentLength += 9; + // account for collection + if ('collection' in v) { + currentLength += v.collection.length + 1; + } + + // account for db if present + if ('db' in v && typeof v.db === 'string') { + currentLength += 8 + v.db.length + 2; + } + + // account for oid if present + if ('oid' in v) { + currentLength += 35; + } } } From befdc475e3e947cd9431f50d7534924a316eedb1 Mon Sep 17 00:00:00 2001 From: Warren James Date: Wed, 22 Jan 2025 16:07:33 -0500 Subject: [PATCH 05/17] update comments --- src/mongo_logger.ts | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/mongo_logger.ts b/src/mongo_logger.ts index d880484be4a..a9349366832 100644 --- a/src/mongo_logger.ts +++ b/src/mongo_logger.ts @@ -481,14 +481,12 @@ export function stringifyWithMaxLen( } else if (value != null && typeof value === 'object' && '_bsontype' in value) { const v = value as BSONObject; if (v._bsontype === 'Binary') { - // This is an estimate based on the fact that the base64 is approximately 1.3x the length of - // the actual binary sequence - // Also accounting for stringified fields before the binary sequence and the fields after - // the binary sequence - currentLength += (value.position + value.position * 0.3 + 22 + 17) | 0; + // '{"$binary":{"base64":"","subType":"XX"}}' + // This is an estimate based on the fact that the base64 is approximately 1.33x the length of + // the actual binary sequence https://en.wikipedia.org/wiki/Base64 + currentLength += (22 + value.position + value.position * 0.33 + 18) | 0; } else if (v._bsontype === 'Code') { // '{"$code":""}' or '{"$code":"","$scope":}' - // TODO: Account for scope? if (v.scope == null) { currentLength += v.code.length + 10 + 2; } else { @@ -496,7 +494,6 @@ export function stringifyWithMaxLen( currentLength += v.code.length + 10 + 11; } } else if (v._bsontype === 'Decimal128') { - // TODO: Is this worth doing here? currentLength += value.toExtendedJSON().length; } else if (v._bsontype === 'Double') { // Doesn't account for representing integers as .0 From 2c255257873ff5942379e35a8e44e8f439433db3 Mon Sep 17 00:00:00 2001 From: Warren James Date: Wed, 22 Jan 2025 17:36:34 -0500 Subject: [PATCH 06/17] update condition to check binary --- src/mongo_logger.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/mongo_logger.ts b/src/mongo_logger.ts index a9349366832..36b28ec080c 100644 --- a/src/mongo_logger.ts +++ b/src/mongo_logger.ts @@ -475,7 +475,12 @@ export function stringifyWithMaxLen( currentLength += String(value).length; } else if (typeof value === 'boolean') { currentLength += value ? 4 : 5; - } else if ('buffer' in value && isUint8Array(value.buffer)) { + } else if ( + value != null && + typeof value === 'object' && + 'buffer' in value && + isUint8Array(value.buffer) + ) { // Handle binData currentLength += (value.buffer.byteLength + value.buffer.byteLength * 0.5) | 0; } else if (value != null && typeof value === 'object' && '_bsontype' in value) { @@ -490,7 +495,7 @@ export function stringifyWithMaxLen( if (v.scope == null) { currentLength += v.code.length + 10 + 2; } else { - // Ignoring actual scope object + // Ignoring actual scope object, so this undercounts currentLength += v.code.length + 10 + 11; } } else if (v._bsontype === 'Decimal128') { @@ -531,6 +536,7 @@ export function stringifyWithMaxLen( ) { currentLength += 19 + String(v.t).length + 5 + String(v.i).length + 2; } else if (v._bsontype === 'DBRef') { + // TODO: Handle fields property; currently undercounts // '{"$ref":"","$id":}' or '{"$ref":"","$id":,"$db":"test"}' currentLength += 9; // account for collection @@ -547,6 +553,8 @@ export function stringifyWithMaxLen( if ('oid' in v) { currentLength += 35; } + } else { + // Unknown BSON type, handle same as plain objects } } From 0e08c227f8b9dcab1cd795ab7901dc806cc49958 Mon Sep 17 00:00:00 2001 From: Warren James Date: Wed, 22 Jan 2025 17:37:08 -0500 Subject: [PATCH 07/17] add wip tests --- test/unit/mongo_logger.test.ts | 196 ++++++++++++++++++++++++++++++++- 1 file changed, 195 insertions(+), 1 deletion(-) diff --git a/test/unit/mongo_logger.test.ts b/test/unit/mongo_logger.test.ts index fff75bf9a21..f25fc90c22b 100644 --- a/test/unit/mongo_logger.test.ts +++ b/test/unit/mongo_logger.test.ts @@ -1,4 +1,16 @@ -import { EJSON, ObjectId } from 'bson'; +import { + BSONRegExp, + BSONSymbol, + Code, + DBRef, + Double, + EJSON, + Int32, + Long, + MaxKey, + MinKey, + ObjectId +} from 'bson'; import { expect } from 'chai'; import * as sinon from 'sinon'; import { Readable, Writable } from 'stream'; @@ -1612,3 +1624,185 @@ describe('class MongoLogger', function () { } }); }); + +describe('stringifyWithMaxLen', function () { + let returnVal: string; + + describe('when stringifying a string field', function () { + it('does not prematurely redact the next key', function () { + const doc = { + a: 'aaa', + b: 'bbb' + }; + + returnVal = stringifyWithMaxLen(doc, 13); + expect(returnVal).to.contain('"b...'); + }); + }); + + describe('when stringifying a number field', function () { + it('does not prematurely redact the next key', function () { + const doc = { + a: 1000, + b: 'bbb' + }; + returnVal = stringifyWithMaxLen(doc, 12); + + expect(returnVal).to.contain('"b...'); + }); + }); + + describe('when stringifying a bigint field', function () { + it('does not prematurely redact the next key', function () { + const doc = { + a: 1000n, + b: 'bbb' + }; + returnVal = stringifyWithMaxLen(doc, 12); + + expect(returnVal).to.contain('"b...'); + }); + }); + + describe('when stringifying a BSON Code field', function () { + it('does not prematurely redact the next key', function () { + const doc = { + c: new Code('console.log();'), + b: 'bbb' + }; + returnVal = stringifyWithMaxLen(doc, 34); + + expect(returnVal).to.contain('"b...'); + }); + }); + + describe('when stringifying a BSON Decimal128 field', function () {}); + + describe('when stringifying a BSON Double field', function () { + it('does not prematurely redact the next key', function () { + const doc = { + c: new Double(123.1), + b: 'bbb' + }; + returnVal = stringifyWithMaxLen(doc, 13); + + expect(returnVal).to.contain('"b...'); + }); + }); + + describe('when stringifying a BSON Int32 field', function () { + it('does not prematurely redact the next key', function () { + const doc = { + c: new Int32(123), + b: 'bbb' + }; + returnVal = stringifyWithMaxLen(doc, 11); + + expect(returnVal).to.contain('"b...'); + }); + }); + + describe('when stringifying a BSON Long field', function () { + it('does not prematurely redact the next key', function () { + const doc = { + c: new Long(123), + b: 'bbb' + }; + returnVal = stringifyWithMaxLen(doc, 11); + + expect(returnVal).to.contain('"b...'); + }); + }); + + describe('when stringifying a BSON MaxKey field', function () { + it('does not prematurely redact the next key', function () { + const doc = { + c: new MaxKey(), + b: 'bbb' + }; + returnVal = stringifyWithMaxLen(doc, 21); + + expect(returnVal).to.contain('"b...'); + }); + }); + + describe('when stringifying a BSON MinKey field', function () { + it('does not prematurely redact the next key', function () { + const doc = { + c: new MinKey(), + b: 'bbb' + }; + returnVal = stringifyWithMaxLen(doc, 21); + + expect(returnVal).to.contain('"b...'); + }); + }); + + describe('when stringifying a BSON ObjectId field', function () { + it('does not prematurely redact the next key', function () { + const doc = { + c: new ObjectId(), + b: 'bbb' + }; + returnVal = stringifyWithMaxLen(doc, 43); + + expect(returnVal).to.contain('"b...'); + }); + }); + + describe('when stringifying a BSON BSONRegExp field', function () { + it('does not prematurely redact the next key', function () { + const doc = { + c: new BSONRegExp('testRegex', 'is'), + b: 'bbb' + }; + returnVal = stringifyWithMaxLen(doc, 69); + + expect(returnVal).to.contain('"b...'); + }); + }); + + describe('when stringifying a BSON BSONSymbol field', function () { + it('does not prematurely redact the next key', function () { + const doc = { + c: new BSONSymbol('testSymbol'), + b: 'bbb' + }; + returnVal = stringifyWithMaxLen(doc, 32); + + expect(returnVal).to.contain('"b...'); + }); + }); + + describe('when stringifying a BSON DBRef field', function () { + describe('when db, collection, oid and fields are defined', function () { + // TODO + }); + + describe('when db, collection and oid are defined', function () { + it('does not prematurely redact the next key', function () { + const oid = new ObjectId(); + const doc = { + c: new DBRef('coll', oid, 'db'), + b: 'bbb' + }; + returnVal = stringifyWithMaxLen(doc, 76); + + expect(returnVal).to.contain('"b...'); + }); + }); + + describe('when collection and oid are defined', function () { + it('does not prematurely redact the next key', function () { + const oid = new ObjectId(); + const doc = { + c: new DBRef('coll', oid), + b: 'bbb' + }; + returnVal = stringifyWithMaxLen(doc, 65); + + expect(returnVal).to.contain('"b...'); + }); + }); + }); +}); From 891cee34fbbbf35298bd8f71955389fb1dd55037 Mon Sep 17 00:00:00 2001 From: Warren James Date: Wed, 22 Jan 2025 18:26:55 -0500 Subject: [PATCH 08/17] refactor to use switch --- src/mongo_logger.ts | 151 ++++++++++++++++++-------------------------- 1 file changed, 63 insertions(+), 88 deletions(-) diff --git a/src/mongo_logger.ts b/src/mongo_logger.ts index 36b28ec080c..6b10ce4cb2f 100644 --- a/src/mongo_logger.ts +++ b/src/mongo_logger.ts @@ -468,96 +468,71 @@ export function stringifyWithMaxLen( // +4 accounts for 2 quotation marks, colon and comma after value currentLength += key.length + 4; - if (typeof value === 'string') { - // +2 accounts for quotes - currentLength += value.length + 2; - } else if (typeof value === 'number' || typeof value === 'bigint') { - currentLength += String(value).length; - } else if (typeof value === 'boolean') { - currentLength += value ? 4 : 5; - } else if ( - value != null && - typeof value === 'object' && - 'buffer' in value && - isUint8Array(value.buffer) - ) { - // Handle binData - currentLength += (value.buffer.byteLength + value.buffer.byteLength * 0.5) | 0; - } else if (value != null && typeof value === 'object' && '_bsontype' in value) { - const v = value as BSONObject; - if (v._bsontype === 'Binary') { - // '{"$binary":{"base64":"","subType":"XX"}}' - // This is an estimate based on the fact that the base64 is approximately 1.33x the length of - // the actual binary sequence https://en.wikipedia.org/wiki/Base64 - currentLength += (22 + value.position + value.position * 0.33 + 18) | 0; - } else if (v._bsontype === 'Code') { - // '{"$code":""}' or '{"$code":"","$scope":}' - if (v.scope == null) { - currentLength += v.code.length + 10 + 2; - } else { - // Ignoring actual scope object, so this undercounts - currentLength += v.code.length + 10 + 11; - } - } else if (v._bsontype === 'Decimal128') { - currentLength += value.toExtendedJSON().length; - } else if (v._bsontype === 'Double') { - // Doesn't account for representing integers as .0 - if ('value' in v && typeof v.value === 'number') currentLength += String(v.value).length; - } else if (v._bsontype === 'Int32') { - if ('value' in v && typeof v.value === 'number') currentLength += String(v.value).length; - } else if (v._bsontype === 'Long') { - if ('toString' in v && typeof v.toString === 'function') { - currentLength += v.toString().length; - } - } else if (v._bsontype === 'MaxKey' || v._bsontype === 'MinKey') { - // '{"$maxKey":1}' or '{"$minKey":1}' - currentLength += 13; - } else if (v._bsontype === 'ObjectId') { - // '{"$oid":"XXXXXXXXXXXXXXXXXXXXXXXX"}' - currentLength += 35; - } else if ( - v._bsontype === 'BSONRegExp' && - 'pattern' in v && - typeof v.pattern === 'string' && - 'options' in v && - typeof v.options === 'string' - ) { - // '{"$regularExpression":{"pattern":"","options":""}}' - currentLength += 34 + v.pattern.length + 13 + v.options.length + 3; - } else if (v._bsontype === 'BSONSymbol' && 'value' in v && v.value === 'string') { - // '{"$symbol": ""}' - currentLength += 12 + v.value.length + 2; - } else if ( - v._bsontype === 'Timestamp' && - 't' in v && - typeof v.t === 'string' && - 'i' in v && - typeof v.i === 'string' - ) { - currentLength += 19 + String(v.t).length + 5 + String(v.i).length + 2; - } else if (v._bsontype === 'DBRef') { - // TODO: Handle fields property; currently undercounts - // '{"$ref":"","$id":}' or '{"$ref":"","$id":,"$db":"test"}' - currentLength += 9; - // account for collection - if ('collection' in v) { - currentLength += v.collection.length + 1; - } - - // account for db if present - if ('db' in v && typeof v.db === 'string') { - currentLength += 8 + v.db.length + 2; - } - - // account for oid if present - if ('oid' in v) { - currentLength += 35; + if (value == null) return value; + + switch (typeof value) { + case 'string': + // +2 accounts for quotes + currentLength += value.length + 2; + break; + case 'number': + case 'bigint': + currentLength += String(value).length; + break; + case 'boolean': + currentLength += value ? 4 : 5; + break; + case 'object': + if ('buffer' in value && isUint8Array(value.buffer)) { + // Handle binData + currentLength += (value.buffer.byteLength + value.buffer.byteLength * 0.5) | 0; + } else if ('_bsontype' in value) { + const v = value as BSONObject; + switch (v._bsontype) { + case 'Int32': + currentLength += String(v.value).length; + break; + case 'Double': + // Doesn't account for representing integers as .0 + currentLength += String(v.value).length; + break; + case 'Long': + currentLength += v.toString().length; + break; + case 'ObjectId': + // '{"$oid":"XXXXXXXXXXXXXXXXXXXXXXXX"}' + currentLength += 35; + break; + case 'MaxKey': + case 'MinKey': + // '{"$maxKey":1}' or '{"$minKey":1}' + currentLength += 13; + break; + case 'Binary': + // '{"$binary":{"base64":"","subType":"XX"}}' + // This is an estimate based on the fact that the base64 is approximately 1.33x the length of + // the actual binary sequence https://en.wikipedia.org/wiki/Base64 + currentLength += (22 + value.position + value.position * 0.33 + 18) | 0; + break; + case 'Timestamp': + currentLength += 19 + String(v.t).length + 5 + String(v.i).length + 2; + break; + case 'Code': + // '{"$code":""}' or '{"$code":"","$scope":}' + if (v.scope == null) { + currentLength += v.code.length + 10 + 2; + } else { + // Ignoring actual scope object, so this undercounts by a significant amount + currentLength += v.code.length + 10 + 11; + } + break; + case 'BSONRegExp': + // '{"$regularExpression":{"pattern":"","options":""}}' + currentLength += 34 + v.pattern.length + 13 + v.options.length + 3; + break; + } } - } else { - // Unknown BSON type, handle same as plain objects - } } - return value; }; From b2fddf18639429908307203fa3b41b24dd0c9802 Mon Sep 17 00:00:00 2001 From: Warren James Date: Thu, 23 Jan 2025 11:41:15 -0500 Subject: [PATCH 09/17] remove unneeded tests --- test/unit/mongo_logger.test.ts | 58 ---------------------------------- 1 file changed, 58 deletions(-) diff --git a/test/unit/mongo_logger.test.ts b/test/unit/mongo_logger.test.ts index f25fc90c22b..b537ab73094 100644 --- a/test/unit/mongo_logger.test.ts +++ b/test/unit/mongo_logger.test.ts @@ -1676,8 +1676,6 @@ describe('stringifyWithMaxLen', function () { }); }); - describe('when stringifying a BSON Decimal128 field', function () {}); - describe('when stringifying a BSON Double field', function () { it('does not prematurely redact the next key', function () { const doc = { @@ -1702,18 +1700,6 @@ describe('stringifyWithMaxLen', function () { }); }); - describe('when stringifying a BSON Long field', function () { - it('does not prematurely redact the next key', function () { - const doc = { - c: new Long(123), - b: 'bbb' - }; - returnVal = stringifyWithMaxLen(doc, 11); - - expect(returnVal).to.contain('"b...'); - }); - }); - describe('when stringifying a BSON MaxKey field', function () { it('does not prematurely redact the next key', function () { const doc = { @@ -1761,48 +1747,4 @@ describe('stringifyWithMaxLen', function () { expect(returnVal).to.contain('"b...'); }); }); - - describe('when stringifying a BSON BSONSymbol field', function () { - it('does not prematurely redact the next key', function () { - const doc = { - c: new BSONSymbol('testSymbol'), - b: 'bbb' - }; - returnVal = stringifyWithMaxLen(doc, 32); - - expect(returnVal).to.contain('"b...'); - }); - }); - - describe('when stringifying a BSON DBRef field', function () { - describe('when db, collection, oid and fields are defined', function () { - // TODO - }); - - describe('when db, collection and oid are defined', function () { - it('does not prematurely redact the next key', function () { - const oid = new ObjectId(); - const doc = { - c: new DBRef('coll', oid, 'db'), - b: 'bbb' - }; - returnVal = stringifyWithMaxLen(doc, 76); - - expect(returnVal).to.contain('"b...'); - }); - }); - - describe('when collection and oid are defined', function () { - it('does not prematurely redact the next key', function () { - const oid = new ObjectId(); - const doc = { - c: new DBRef('coll', oid), - b: 'bbb' - }; - returnVal = stringifyWithMaxLen(doc, 65); - - expect(returnVal).to.contain('"b...'); - }); - }); - }); }); From 2876bdb5fc3b0bad93a9cbf39fb36b9fe83d9146 Mon Sep 17 00:00:00 2001 From: Warren James Date: Thu, 23 Jan 2025 17:54:02 -0500 Subject: [PATCH 10/17] Update test/unit/mongo_logger.test.ts Co-authored-by: Neal Beeken --- test/unit/mongo_logger.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/test/unit/mongo_logger.test.ts b/test/unit/mongo_logger.test.ts index b537ab73094..846309c0703 100644 --- a/test/unit/mongo_logger.test.ts +++ b/test/unit/mongo_logger.test.ts @@ -1,6 +1,5 @@ import { BSONRegExp, - BSONSymbol, Code, DBRef, Double, From 4b9cae0692eeb263af5e6569ae61416e46e9672a Mon Sep 17 00:00:00 2001 From: Warren James Date: Thu, 23 Jan 2025 17:54:13 -0500 Subject: [PATCH 11/17] Update test/unit/mongo_logger.test.ts Co-authored-by: Neal Beeken --- test/unit/mongo_logger.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/test/unit/mongo_logger.test.ts b/test/unit/mongo_logger.test.ts index 846309c0703..b11c3100265 100644 --- a/test/unit/mongo_logger.test.ts +++ b/test/unit/mongo_logger.test.ts @@ -1,7 +1,6 @@ import { BSONRegExp, Code, - DBRef, Double, EJSON, Int32, From 52f2a4a94442f3eff2ef19a18b8011b43f0286da Mon Sep 17 00:00:00 2001 From: Warren James Date: Thu, 23 Jan 2025 17:54:25 -0500 Subject: [PATCH 12/17] Update test/unit/mongo_logger.test.ts Co-authored-by: Neal Beeken --- test/unit/mongo_logger.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/test/unit/mongo_logger.test.ts b/test/unit/mongo_logger.test.ts index b11c3100265..fd54c3fc4a5 100644 --- a/test/unit/mongo_logger.test.ts +++ b/test/unit/mongo_logger.test.ts @@ -4,7 +4,6 @@ import { Double, EJSON, Int32, - Long, MaxKey, MinKey, ObjectId From d99e3b7b1fc92c58d3ffb81db3591df8bfda867e Mon Sep 17 00:00:00 2001 From: Warren James Date: Thu, 23 Jan 2025 18:14:47 -0500 Subject: [PATCH 13/17] address review comments --- src/mongo_logger.ts | 6 ++++-- test/unit/mongo_logger.test.ts | 22 ++++++++++------------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/mongo_logger.ts b/src/mongo_logger.ts index 6b10ce4cb2f..db38d1a2ad5 100644 --- a/src/mongo_logger.ts +++ b/src/mongo_logger.ts @@ -493,8 +493,9 @@ export function stringifyWithMaxLen( currentLength += String(v.value).length; break; case 'Double': - // Doesn't account for representing integers as .0 - currentLength += String(v.value).length; + // Account for representing integers as .0 + currentLength += + (v.value | 0) === v.value ? String(v.value).length + 2 : String(v.value).length; break; case 'Long': currentLength += v.toString().length; @@ -515,6 +516,7 @@ export function stringifyWithMaxLen( currentLength += (22 + value.position + value.position * 0.33 + 18) | 0; break; case 'Timestamp': + // '{"$timestamp":{"t":,"i":}}' currentLength += 19 + String(v.t).length + 5 + String(v.i).length + 2; break; case 'Code': diff --git a/test/unit/mongo_logger.test.ts b/test/unit/mongo_logger.test.ts index fd54c3fc4a5..27b783f3492 100644 --- a/test/unit/mongo_logger.test.ts +++ b/test/unit/mongo_logger.test.ts @@ -1623,8 +1623,6 @@ describe('class MongoLogger', function () { }); describe('stringifyWithMaxLen', function () { - let returnVal: string; - describe('when stringifying a string field', function () { it('does not prematurely redact the next key', function () { const doc = { @@ -1632,7 +1630,7 @@ describe('stringifyWithMaxLen', function () { b: 'bbb' }; - returnVal = stringifyWithMaxLen(doc, 13); + const returnVal = stringifyWithMaxLen(doc, 13); expect(returnVal).to.contain('"b...'); }); }); @@ -1643,7 +1641,7 @@ describe('stringifyWithMaxLen', function () { a: 1000, b: 'bbb' }; - returnVal = stringifyWithMaxLen(doc, 12); + const returnVal = stringifyWithMaxLen(doc, 12); expect(returnVal).to.contain('"b...'); }); @@ -1655,7 +1653,7 @@ describe('stringifyWithMaxLen', function () { a: 1000n, b: 'bbb' }; - returnVal = stringifyWithMaxLen(doc, 12); + const returnVal = stringifyWithMaxLen(doc, 12); expect(returnVal).to.contain('"b...'); }); @@ -1667,7 +1665,7 @@ describe('stringifyWithMaxLen', function () { c: new Code('console.log();'), b: 'bbb' }; - returnVal = stringifyWithMaxLen(doc, 34); + const returnVal = stringifyWithMaxLen(doc, 34); expect(returnVal).to.contain('"b...'); }); @@ -1679,7 +1677,7 @@ describe('stringifyWithMaxLen', function () { c: new Double(123.1), b: 'bbb' }; - returnVal = stringifyWithMaxLen(doc, 13); + const returnVal = stringifyWithMaxLen(doc, 13); expect(returnVal).to.contain('"b...'); }); @@ -1691,7 +1689,7 @@ describe('stringifyWithMaxLen', function () { c: new Int32(123), b: 'bbb' }; - returnVal = stringifyWithMaxLen(doc, 11); + const returnVal = stringifyWithMaxLen(doc, 11); expect(returnVal).to.contain('"b...'); }); @@ -1703,7 +1701,7 @@ describe('stringifyWithMaxLen', function () { c: new MaxKey(), b: 'bbb' }; - returnVal = stringifyWithMaxLen(doc, 21); + const returnVal = stringifyWithMaxLen(doc, 21); expect(returnVal).to.contain('"b...'); }); @@ -1715,7 +1713,7 @@ describe('stringifyWithMaxLen', function () { c: new MinKey(), b: 'bbb' }; - returnVal = stringifyWithMaxLen(doc, 21); + const returnVal = stringifyWithMaxLen(doc, 21); expect(returnVal).to.contain('"b...'); }); @@ -1727,7 +1725,7 @@ describe('stringifyWithMaxLen', function () { c: new ObjectId(), b: 'bbb' }; - returnVal = stringifyWithMaxLen(doc, 43); + const returnVal = stringifyWithMaxLen(doc, 43); expect(returnVal).to.contain('"b...'); }); @@ -1739,7 +1737,7 @@ describe('stringifyWithMaxLen', function () { c: new BSONRegExp('testRegex', 'is'), b: 'bbb' }; - returnVal = stringifyWithMaxLen(doc, 69); + const returnVal = stringifyWithMaxLen(doc, 69); expect(returnVal).to.contain('"b...'); }); From 35fc08879d275134f440f25bcdfa989811421db6 Mon Sep 17 00:00:00 2001 From: Warren James Date: Fri, 24 Jan 2025 13:51:06 -0500 Subject: [PATCH 14/17] address review --- src/mongo_logger.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/mongo_logger.ts b/src/mongo_logger.ts index db38d1a2ad5..588542da697 100644 --- a/src/mongo_logger.ts +++ b/src/mongo_logger.ts @@ -466,6 +466,8 @@ export function stringifyWithMaxLen( } // +4 accounts for 2 quotation marks, colon and comma after value + // Note that this potentially undercounts since it does not account for escape sequences which + // will have an additional backslash added to them once passed through JSON.stringify. currentLength += key.length + 4; if (value == null) return value; @@ -473,6 +475,7 @@ export function stringifyWithMaxLen( switch (typeof value) { case 'string': // +2 accounts for quotes + // Note that this potentially undercounts similarly to the key length calculation currentLength += value.length + 2; break; case 'number': @@ -483,9 +486,9 @@ export function stringifyWithMaxLen( currentLength += value ? 4 : 5; break; case 'object': - if ('buffer' in value && isUint8Array(value.buffer)) { + if (isUint8Array(value)) { // Handle binData - currentLength += (value.buffer.byteLength + value.buffer.byteLength * 0.5) | 0; + currentLength += (value.byteLength + value.byteLength * 0.5) | 0; } else if ('_bsontype' in value) { const v = value as BSONObject; switch (v._bsontype) { From 52c6f16e897d16f024d7c675a30ccbbc961562b9 Mon Sep 17 00:00:00 2001 From: Warren James Date: Fri, 24 Jan 2025 13:52:04 -0500 Subject: [PATCH 15/17] test lint --- test/unit/mongo_logger.test.ts | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/test/unit/mongo_logger.test.ts b/test/unit/mongo_logger.test.ts index 27b783f3492..da47bd8399c 100644 --- a/test/unit/mongo_logger.test.ts +++ b/test/unit/mongo_logger.test.ts @@ -1,13 +1,4 @@ -import { - BSONRegExp, - Code, - Double, - EJSON, - Int32, - MaxKey, - MinKey, - ObjectId -} from 'bson'; +import { BSONRegExp, Code, Double, EJSON, Int32, MaxKey, MinKey, ObjectId } from 'bson'; import { expect } from 'chai'; import * as sinon from 'sinon'; import { Readable, Writable } from 'stream'; From 1019f5ecdcd1ac5813da4b609ae401c3dada4ca7 Mon Sep 17 00:00:00 2001 From: Warren James Date: Fri, 24 Jan 2025 13:52:34 -0500 Subject: [PATCH 16/17] remove hybrid uint8array/binary case --- src/mongo_logger.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/mongo_logger.ts b/src/mongo_logger.ts index 588542da697..2cac06f1b26 100644 --- a/src/mongo_logger.ts +++ b/src/mongo_logger.ts @@ -486,10 +486,7 @@ export function stringifyWithMaxLen( currentLength += value ? 4 : 5; break; case 'object': - if (isUint8Array(value)) { - // Handle binData - currentLength += (value.byteLength + value.byteLength * 0.5) | 0; - } else if ('_bsontype' in value) { + if ('_bsontype' in value) { const v = value as BSONObject; switch (v._bsontype) { case 'Int32': From 7e70174d0ca7eb9294e3c874e92b63cf20cb0b66 Mon Sep 17 00:00:00 2001 From: Warren James Date: Fri, 24 Jan 2025 14:18:47 -0500 Subject: [PATCH 17/17] handle Uint8Array case --- src/mongo_logger.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/mongo_logger.ts b/src/mongo_logger.ts index 2cac06f1b26..c6e581c59e8 100644 --- a/src/mongo_logger.ts +++ b/src/mongo_logger.ts @@ -486,7 +486,12 @@ export function stringifyWithMaxLen( currentLength += value ? 4 : 5; break; case 'object': - if ('_bsontype' in value) { + if (isUint8Array(value)) { + // '{"$binary":{"base64":"","subType":"XX"}}' + // This is an estimate based on the fact that the base64 is approximately 1.33x the length of + // the actual binary sequence https://en.wikipedia.org/wiki/Base64 + currentLength += (22 + value.byteLength + value.byteLength * 0.33 + 18) | 0; + } else if ('_bsontype' in value) { const v = value as BSONObject; switch (v._bsontype) { case 'Int32':