From 24bc4830102efbfa90326b1696056c832226f1d0 Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Thu, 21 Dec 2023 14:58:37 -0500 Subject: [PATCH 01/21] add remote config support for custom tags (#3875) --- .../src/appsec/remote_config/capabilities.js | 6 +- .../src/appsec/remote_config/index.js | 4 + packages/dd-trace/src/config.js | 138 ++++++++++-------- packages/dd-trace/src/opentracing/tracer.js | 4 +- packages/dd-trace/src/telemetry/index.js | 29 +++- .../test/appsec/remote_config/index.spec.js | 117 +++++++-------- packages/dd-trace/test/config.spec.js | 2 +- 7 files changed, 169 insertions(+), 131 deletions(-) diff --git a/packages/dd-trace/src/appsec/remote_config/capabilities.js b/packages/dd-trace/src/appsec/remote_config/capabilities.js index 6032b6543c2..8d53eb05596 100644 --- a/packages/dd-trace/src/appsec/remote_config/capabilities.js +++ b/packages/dd-trace/src/appsec/remote_config/capabilities.js @@ -10,5 +10,9 @@ module.exports = { ASM_CUSTOM_RULES: 1n << 8n, ASM_CUSTOM_BLOCKING_RESPONSE: 1n << 9n, ASM_TRUSTED_IPS: 1n << 10n, - ASM_API_SECURITY_SAMPLE_RATE: 1n << 11n + ASM_API_SECURITY_SAMPLE_RATE: 1n << 11n, + APM_TRACING_SAMPLE_RATE: 1n << 12n, + APM_TRACING_LOGS_INJECTION: 1n << 13n, + APM_TRACING_HTTP_HEADER_TAGS: 1n << 14n, + APM_TRACING_CUSTOM_TAGS: 1n << 15n } diff --git a/packages/dd-trace/src/appsec/remote_config/index.js b/packages/dd-trace/src/appsec/remote_config/index.js index 08d912d22e1..e2b45ff158f 100644 --- a/packages/dd-trace/src/appsec/remote_config/index.js +++ b/packages/dd-trace/src/appsec/remote_config/index.js @@ -10,6 +10,10 @@ let rc function enable (config) { rc = new RemoteConfigManager(config) + rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_CUSTOM_TAGS, true) + rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_HTTP_HEADER_TAGS, true) + rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_LOGS_INJECTION, true) + rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_SAMPLE_RATE, true) const activation = Activation.fromConfig(config) diff --git a/packages/dd-trace/src/config.js b/packages/dd-trace/src/config.js index fb5615bdf17..40c9c68d091 100644 --- a/packages/dd-trace/src/config.js +++ b/packages/dd-trace/src/config.js @@ -109,13 +109,6 @@ class Config { log.use(this.logger) log.toggle(this.debug, this.logLevel, this) - this.tags = {} - - tagger.add(this.tags, process.env.DD_TAGS) - tagger.add(this.tags, process.env.DD_TRACE_TAGS) - tagger.add(this.tags, process.env.DD_TRACE_GLOBAL_TAGS) - tagger.add(this.tags, options.tags) - const DD_TRACING_ENABLED = coalesce( process.env.DD_TRACING_ENABLED, true @@ -184,33 +177,12 @@ class Config { false ) - const DD_SERVICE = options.service || - process.env.DD_SERVICE || - process.env.DD_SERVICE_NAME || - this.tags.service || - process.env.AWS_LAMBDA_FUNCTION_NAME || - process.env.FUNCTION_NAME || // Google Cloud Function Name set by deprecated runtimes - process.env.K_SERVICE || // Google Cloud Function Name set by newer runtimes - process.env.WEBSITE_SITE_NAME || // set by Azure Functions - pkg.name || - 'node' const DD_SERVICE_MAPPING = coalesce( options.serviceMapping, process.env.DD_SERVICE_MAPPING ? fromEntries( process.env.DD_SERVICE_MAPPING.split(',').map(x => x.trim().split(':')) ) : {} ) - const DD_ENV = coalesce( - options.env, - process.env.DD_ENV, - this.tags.env - ) - const DD_VERSION = coalesce( - options.version, - process.env.DD_VERSION, - this.tags.version, - pkg.version - ) const DD_TRACE_STARTUP_LOGS = coalesce( options.startupLogs, process.env.DD_TRACE_STARTUP_LOGS, @@ -583,7 +555,6 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?) this.dsmEnabled = isTrue(DD_DATA_STREAMS_ENABLED) this.openAiLogsEnabled = DD_OPENAI_LOGS_ENABLED this.apiKey = DD_API_KEY - this.env = DD_ENV this.url = DD_CIVISIBILITY_AGENTLESS_URL ? new URL(DD_CIVISIBILITY_AGENTLESS_URL) : getAgentUrl(DD_TRACE_AGENT_URL, options) this.site = coalesce(options.site, process.env.DD_SITE, 'datadoghq.com') @@ -595,9 +566,7 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?) this.clientIpEnabled = DD_TRACE_CLIENT_IP_ENABLED this.clientIpHeader = DD_TRACE_CLIENT_IP_HEADER this.plugins = !!coalesce(options.plugins, true) - this.service = DD_SERVICE this.serviceMapping = DD_SERVICE_MAPPING - this.version = DD_VERSION this.dogstatsd = { hostname: coalesce(dogstatsd.hostname, process.env.DD_DOGSTATSD_HOSTNAME, this.hostname), port: String(coalesce(dogstatsd.port, process.env.DD_DOGSTATSD_PORT, 8125)) @@ -690,6 +659,31 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?) // Requires an accompanying DD_APM_OBFUSCATION_MEMCACHED_KEEP_COMMAND=true in the agent this.memcachedCommandEnabled = isTrue(DD_TRACE_MEMCACHED_COMMAND_ENABLED) + this.stats = { + enabled: isTrue(DD_TRACE_STATS_COMPUTATION_ENABLED) + } + + this.traceId128BitGenerationEnabled = isTrue(DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED) + this.traceId128BitLoggingEnabled = isTrue(DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED) + + this.isGCPFunction = isGCPFunction + this.isAzureFunctionConsumptionPlan = isAzureFunctionConsumptionPlan + + this.spanLeakDebug = Number(DD_TRACE_SPAN_LEAK_DEBUG) + + this._applyDefaults() + this._applyEnvironment() + this._applyOptions(options) + this._applyRemote({}) + this._merge() + + tagger.add(this.tags, { + service: this.service, + env: this.env, + version: this.version, + 'runtime-id': uuid() + }) + if (this.gitMetadataEnabled) { this.repositoryUrl = removeUserSensitiveInfo( coalesce( @@ -722,31 +716,6 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?) } } } - - this.stats = { - enabled: isTrue(DD_TRACE_STATS_COMPUTATION_ENABLED) - } - - this.traceId128BitGenerationEnabled = isTrue(DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED) - this.traceId128BitLoggingEnabled = isTrue(DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED) - - this.isGCPFunction = isGCPFunction - this.isAzureFunctionConsumptionPlan = isAzureFunctionConsumptionPlan - - this.spanLeakDebug = Number(DD_TRACE_SPAN_LEAK_DEBUG) - - tagger.add(this.tags, { - service: this.service, - env: this.env, - version: this.version, - 'runtime-id': uuid() - }) - - this._applyDefaults() - this._applyEnvironment() - this._applyOptions(options) - this._applyRemote({}) - this._merge() } // Supports only a subset of options for now. @@ -761,48 +730,93 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?) } _applyDefaults () { + const { + AWS_LAMBDA_FUNCTION_NAME, + FUNCTION_NAME, + K_SERVICE, + WEBSITE_SITE_NAME + } = process.env + + const service = AWS_LAMBDA_FUNCTION_NAME || + FUNCTION_NAME || // Google Cloud Function Name set by deprecated runtimes + K_SERVICE || // Google Cloud Function Name set by newer runtimes + WEBSITE_SITE_NAME || // set by Azure Functions + pkg.name || + 'node' + const defaults = this._defaults = {} + this._setValue(defaults, 'service', service) + this._setValue(defaults, 'env', undefined) + this._setValue(defaults, 'version', pkg.version) this._setUnit(defaults, 'sampleRate', undefined) this._setBoolean(defaults, 'logInjection', false) this._setArray(defaults, 'headerTags', []) + this._setValue(defaults, 'tags', {}) } _applyEnvironment () { const { - DD_TRACE_SAMPLE_RATE, + DD_ENV, DD_LOGS_INJECTION, - DD_TRACE_HEADER_TAGS + DD_SERVICE, + DD_SERVICE_NAME, + DD_TAGS, + DD_TRACE_GLOBAL_TAGS, + DD_TRACE_HEADER_TAGS, + DD_TRACE_SAMPLE_RATE, + DD_TRACE_TAGS, + DD_VERSION } = process.env + const tags = {} const env = this._env = {} + tagger.add(tags, DD_TAGS) + tagger.add(tags, DD_TRACE_TAGS) + tagger.add(tags, DD_TRACE_GLOBAL_TAGS) + + this._setValue(env, 'service', DD_SERVICE || DD_SERVICE_NAME || tags.service) + this._setValue(env, 'env', DD_ENV || tags.env) + this._setValue(env, 'version', DD_VERSION || tags.version) this._setUnit(env, 'sampleRate', DD_TRACE_SAMPLE_RATE) this._setBoolean(env, 'logInjection', DD_LOGS_INJECTION) this._setArray(env, 'headerTags', DD_TRACE_HEADER_TAGS) + this._setTags(env, 'tags', tags) } _applyOptions (options) { const opts = this._options = this._options || {} + const tags = {} options = Object.assign({ ingestion: {} }, options, opts) + tagger.add(tags, options.tags) + + this._setValue(opts, 'service', options.service || tags.service) + this._setValue(opts, 'env', options.env || tags.env) + this._setValue(opts, 'version', options.version || tags.version) this._setUnit(opts, 'sampleRate', coalesce(options.sampleRate, options.ingestion.sampleRate)) this._setBoolean(opts, 'logInjection', options.logInjection) this._setArray(opts, 'headerTags', options.headerTags) + this._setTags(opts, 'tags', tags) } _applyRemote (options) { const opts = this._remote = this._remote || {} + const tags = {} const headerTags = options.tracing_header_tags ? options.tracing_header_tags.map(tag => { return tag.tag_name ? `${tag.header}:${tag.tag_name}` : tag.header }) : undefined + tagger.add(tags, options.tracing_tags) + this._setUnit(opts, 'sampleRate', options.tracing_sampling_rate) this._setBoolean(opts, 'logInjection', options.log_injection_enabled) this._setArray(opts, 'headerTags', headerTags) + this._setTags(opts, 'tags', tags) } _setBoolean (obj, name, value) { @@ -842,6 +856,14 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?) } } + _setTags (obj, name, value) { + if (!value || Object.keys(value).length === 0) { + return this._setValue(obj, name, null) + } + + this._setValue(obj, name, value) + } + _setValue (obj, name, value) { obj[name] = value } diff --git a/packages/dd-trace/src/opentracing/tracer.js b/packages/dd-trace/src/opentracing/tracer.js index 2a46d8a8c9a..8b18938631b 100644 --- a/packages/dd-trace/src/opentracing/tracer.js +++ b/packages/dd-trace/src/opentracing/tracer.js @@ -22,10 +22,10 @@ class DatadogTracer { constructor (config) { const Exporter = getExporter(config.experimental.exporter) + this._config = config this._service = config.service this._version = config.version this._env = config.env - this._tags = config.tags this._logInjection = config.logInjection this._debug = config.debug this._prioritySampler = new PrioritySampler(config.env, config.sampler) @@ -64,7 +64,7 @@ class DatadogTracer { integrationName: options.integrationName }, this._debug) - span.addTags(this._tags) + span.addTags(this._config.tags) span.addTags(options.tags) return span diff --git a/packages/dd-trace/src/telemetry/index.js b/packages/dd-trace/src/telemetry/index.js index a99aba775eb..a2db5e13971 100644 --- a/packages/dd-trace/src/telemetry/index.js +++ b/packages/dd-trace/src/telemetry/index.js @@ -286,11 +286,30 @@ function updateConfig (changes, config) { const application = createAppObject(config) const host = createHostObject() - const configuration = changes.map(change => ({ - name: change.name, - value: Array.isArray(change.value) ? change.value.join(',') : change.value, - origin: change.origin - })) + const names = { + sampleRate: 'DD_TRACE_SAMPLE_RATE', + logInjection: 'DD_LOG_INJECTION', + headerTags: 'DD_TRACE_HEADER_TAGS', + tags: 'DD_TAGS' + } + + const configuration = [] + + for (const change of changes) { + if (!names.hasOwnProperty(change.name)) continue + + const name = names[change.name] + const { origin, value } = change + const entry = { name, origin, value } + + if (Array.isArray(value)) { + entry.value = value.join(',') + } else if (name === 'DD_TAGS') { + entry.value = Object.entries(value).map(([key, value]) => `${key}:${value}`) + } + + configuration.push(entry) + } const { reqType, payload } = createPayload('app-client-configuration-change', { configuration }) diff --git a/packages/dd-trace/test/appsec/remote_config/index.spec.js b/packages/dd-trace/test/appsec/remote_config/index.spec.js index 6287c693ddc..9efc142c02e 100644 --- a/packages/dd-trace/test/appsec/remote_config/index.spec.js +++ b/packages/dd-trace/test/appsec/remote_config/index.spec.js @@ -56,8 +56,8 @@ describe('Remote Config index', () => { remoteConfig.enable(config) expect(RemoteConfigManager).to.have.been.calledOnceWithExactly(config) - expect(rc.updateCapabilities).to.have.been.calledOnceWithExactly(RemoteConfigCapabilities.ASM_ACTIVATION, true) - expect(rc.on).to.have.been.calledOnceWith('ASM_FEATURES') + expect(rc.updateCapabilities).to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_ACTIVATION, true) + expect(rc.on).to.have.been.calledWith('ASM_FEATURES') expect(rc.on.firstCall.args[1]).to.be.a('function') }) @@ -67,7 +67,7 @@ describe('Remote Config index', () => { remoteConfig.enable(config) expect(RemoteConfigManager).to.have.been.calledOnceWithExactly(config) - expect(rc.updateCapabilities).to.not.have.been.called + expect(rc.updateCapabilities).to.not.have.been.calledWith('ASM_ACTIVATION') expect(rc.on).to.have.been.calledOnceWith('ASM_FEATURES') expect(rc.on.firstCall.args[1]).to.be.a('function') }) @@ -78,7 +78,7 @@ describe('Remote Config index', () => { remoteConfig.enable(config) expect(RemoteConfigManager).to.have.been.calledOnceWithExactly(config) - expect(rc.updateCapabilities).to.not.have.been.called + expect(rc.updateCapabilities).to.not.have.been.calledWith(RemoteConfigCapabilities.ASM_ACTIVATION, true) expect(rc.on).to.not.have.been.called }) @@ -88,10 +88,9 @@ describe('Remote Config index', () => { remoteConfig.enable(config) expect(RemoteConfigManager).to.have.been.calledOnceWithExactly(config) - expect(rc.updateCapabilities).to.have.been.calledTwice - expect(rc.updateCapabilities.firstCall) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_ACTIVATION, true) - expect(rc.updateCapabilities.secondCall) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_API_SECURITY_SAMPLE_RATE, true) }) @@ -101,8 +100,7 @@ describe('Remote Config index', () => { remoteConfig.enable(config) expect(RemoteConfigManager).to.have.been.calledOnceWithExactly(config) - expect(rc.updateCapabilities).to.have.been.calledOnce - expect(rc.updateCapabilities.firstCall) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_API_SECURITY_SAMPLE_RATE, true) }) @@ -259,7 +257,7 @@ describe('Remote Config index', () => { remoteConfig.enable(config) remoteConfig.enableWafUpdate(config.appsec) - expect(rc.updateCapabilities).to.not.have.been.called + expect(rc.updateCapabilities).to.not.have.been.calledWith('ASM_ACTIVATION') expect(rc.on).to.have.been.called }) @@ -268,30 +266,27 @@ describe('Remote Config index', () => { remoteConfig.enable(config) remoteConfig.enableWafUpdate(config.appsec) - expect(rc.updateCapabilities.callCount).to.be.equal(8) - expect(rc.updateCapabilities.getCall(0)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_IP_BLOCKING, true) - expect(rc.updateCapabilities.getCall(1)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_USER_BLOCKING, true) - expect(rc.updateCapabilities.getCall(2)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_DD_RULES, true) - expect(rc.updateCapabilities.getCall(3)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_EXCLUSIONS, true) - expect(rc.updateCapabilities.getCall(4)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_REQUEST_BLOCKING, true) - expect(rc.updateCapabilities.getCall(5)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_CUSTOM_RULES, true) - expect(rc.updateCapabilities.getCall(6)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_CUSTOM_BLOCKING_RESPONSE, true) - expect(rc.updateCapabilities.getCall(7)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_TRUSTED_IPS, true) - expect(rc.on.callCount).to.be.equal(5) - expect(rc.on.getCall(0)).to.have.been.calledWith('ASM_FEATURES') - expect(rc.on.getCall(1)).to.have.been.calledWith('ASM_DATA') - expect(rc.on.getCall(2)).to.have.been.calledWith('ASM_DD') - expect(rc.on.getCall(3)).to.have.been.calledWith('ASM') - expect(rc.on.getCall(4)).to.have.been.calledWithExactly(kPreUpdate, RuleManager.updateWafFromRC) + expect(rc.on).to.have.been.calledWith('ASM_DATA') + expect(rc.on).to.have.been.calledWith('ASM_DD') + expect(rc.on).to.have.been.calledWith('ASM') + expect(rc.on).to.have.been.calledWithExactly(kPreUpdate, RuleManager.updateWafFromRC) }) it('should activate if appsec is manually enabled', () => { @@ -299,30 +294,27 @@ describe('Remote Config index', () => { remoteConfig.enable(config) remoteConfig.enableWafUpdate(config.appsec) - expect(rc.updateCapabilities.callCount).to.be.equal(8) - expect(rc.updateCapabilities.getCall(0)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_IP_BLOCKING, true) - expect(rc.updateCapabilities.getCall(1)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_USER_BLOCKING, true) - expect(rc.updateCapabilities.getCall(2)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_DD_RULES, true) - expect(rc.updateCapabilities.getCall(3)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_EXCLUSIONS, true) - expect(rc.updateCapabilities.getCall(4)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_REQUEST_BLOCKING, true) - expect(rc.updateCapabilities.getCall(5)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_CUSTOM_RULES, true) - expect(rc.updateCapabilities.getCall(6)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_CUSTOM_BLOCKING_RESPONSE, true) - expect(rc.updateCapabilities.getCall(7)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_TRUSTED_IPS, true) - expect(rc.on.callCount).to.be.equal(5) - expect(rc.on.getCall(0)).to.have.been.calledWith('ASM_FEATURES') - expect(rc.on.getCall(1)).to.have.been.calledWith('ASM_DATA') - expect(rc.on.getCall(2)).to.have.been.calledWith('ASM_DD') - expect(rc.on.getCall(3)).to.have.been.calledWith('ASM') - expect(rc.on.getCall(4)).to.have.been.calledWithExactly(kPreUpdate, RuleManager.updateWafFromRC) + expect(rc.on).to.have.been.calledWith('ASM_DATA') + expect(rc.on).to.have.been.calledWith('ASM_DD') + expect(rc.on).to.have.been.calledWith('ASM') + expect(rc.on).to.have.been.calledWithExactly(kPreUpdate, RuleManager.updateWafFromRC) }) it('should activate if appsec enabled is not defined', () => { @@ -330,24 +322,23 @@ describe('Remote Config index', () => { remoteConfig.enable(config) remoteConfig.enableWafUpdate(config.appsec) - expect(rc.updateCapabilities.callCount).to.be.equal(9) - expect(rc.updateCapabilities.getCall(0)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_ACTIVATION, true) - expect(rc.updateCapabilities.getCall(1)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_IP_BLOCKING, true) - expect(rc.updateCapabilities.getCall(2)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_USER_BLOCKING, true) - expect(rc.updateCapabilities.getCall(3)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_DD_RULES, true) - expect(rc.updateCapabilities.getCall(4)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_EXCLUSIONS, true) - expect(rc.updateCapabilities.getCall(5)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_REQUEST_BLOCKING, true) - expect(rc.updateCapabilities.getCall(6)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_CUSTOM_RULES, true) - expect(rc.updateCapabilities.getCall(7)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_CUSTOM_BLOCKING_RESPONSE, true) - expect(rc.updateCapabilities.getCall(8)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_TRUSTED_IPS, true) }) }) @@ -358,29 +349,27 @@ describe('Remote Config index', () => { rc.updateCapabilities.resetHistory() remoteConfig.disableWafUpdate() - expect(rc.updateCapabilities.callCount).to.be.equal(8) - expect(rc.updateCapabilities.getCall(0)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_IP_BLOCKING, false) - expect(rc.updateCapabilities.getCall(1)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_USER_BLOCKING, false) - expect(rc.updateCapabilities.getCall(2)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_DD_RULES, false) - expect(rc.updateCapabilities.getCall(3)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_EXCLUSIONS, false) - expect(rc.updateCapabilities.getCall(4)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_REQUEST_BLOCKING, false) - expect(rc.updateCapabilities.getCall(5)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_CUSTOM_RULES, false) - expect(rc.updateCapabilities.getCall(6)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_CUSTOM_BLOCKING_RESPONSE, false) - expect(rc.updateCapabilities.getCall(7)) + expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_TRUSTED_IPS, false) - expect(rc.off.callCount).to.be.equal(4) - expect(rc.off.getCall(0)).to.have.been.calledWith('ASM_DATA') - expect(rc.off.getCall(1)).to.have.been.calledWith('ASM_DD') - expect(rc.off.getCall(2)).to.have.been.calledWith('ASM') - expect(rc.off.getCall(3)).to.have.been.calledWithExactly(kPreUpdate, RuleManager.updateWafFromRC) + expect(rc.off).to.have.been.calledWith('ASM_DATA') + expect(rc.off).to.have.been.calledWith('ASM_DD') + expect(rc.off).to.have.been.calledWith('ASM') + expect(rc.off).to.have.been.calledWithExactly(kPreUpdate, RuleManager.updateWafFromRC) }) }) }) diff --git a/packages/dd-trace/test/config.spec.js b/packages/dd-trace/test/config.spec.js index a7647e395f1..21047b5c894 100644 --- a/packages/dd-trace/test/config.spec.js +++ b/packages/dd-trace/test/config.spec.js @@ -763,7 +763,7 @@ describe('Config', () => { expect(config).to.have.property('clientIpHeader', 'x-true-client-ip') expect(config).to.have.property('traceId128BitGenerationEnabled', false) expect(config).to.have.property('traceId128BitLoggingEnabled', false) - expect(config.tags).to.include({ foo: 'foo', baz: 'qux' }) + expect(config.tags).to.include({ foo: 'foo' }) expect(config.tags).to.include({ service: 'test', version: '1.0.0', env: 'development' }) expect(config).to.have.deep.property('serviceMapping', { b: 'bb' }) expect(config).to.have.property('spanAttributeSchema', 'v1') From b2a15f775ede9fd87ac3e59c81460845e2cd7854 Mon Sep 17 00:00:00 2001 From: Brian Devins-Suresh Date: Thu, 21 Dec 2023 16:45:24 -0500 Subject: [PATCH 02/21] Update test agent configuration (#3372) * Update test agent configuration * Update test sgent config * Update test agent configuration --------- Co-authored-by: William Conti --- .github/workflows/plugins.yml | 8 ++++---- docker-compose.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml index 6ba2e693766..59f2dcd3934 100644 --- a/.github/workflows/plugins.yml +++ b/.github/workflows/plugins.yml @@ -30,11 +30,11 @@ jobs: ports: - 3000:3000 testagent: - image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:latest + image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.16.0 env: LOG_LEVEL: DEBUG TRACE_LANGUAGE: javascript - DISABLED_CHECKS: trace_content_length + ENABLED_CHECKS: trace_stall,meta_tracer_version_header,trace_count_header,trace_peer_service PORT: 9126 ports: - 9126:9126 @@ -985,11 +985,11 @@ jobs: - 1521:1521 - 5500:5500 testagent: - image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:latest + image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.16.0 env: LOG_LEVEL: DEBUG TRACE_LANGUAGE: javascript - DISABLED_CHECKS: trace_content_length + ENABLED_CHECKS: trace_stall,meta_tracer_version_header,trace_count_header,trace_peer_service PORT: 9126 ports: - 9126:9126 diff --git a/docker-compose.yml b/docker-compose.yml index 6abe59c677d..226b8ada2af 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -159,11 +159,11 @@ services: - LDAP_PASSWORDS=password1,password2 testagent: - image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.13.1 + image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.16.0 ports: - "127.0.0.1:9126:9126" environment: - LOG_LEVEL=DEBUG - TRACE_LANGUAGE=javascript - - DISABLED_CHECKS=trace_content_length + - ENABLED_CHECKS=trace_stall,meta_tracer_version_header,trace_count_header,trace_peer_service - PORT=9126 From 0796432f815c1cdac828f0707ee800b87fa33629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Fri, 22 Dec 2023 15:04:19 +0100 Subject: [PATCH 03/21] =?UTF-8?q?[ci-visibility]=C2=A0CI=20Visibility=20te?= =?UTF-8?q?lemetry=20-=20agentful=20=20(#3752)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/datadog-instrumentations/src/jest.js | 2 +- packages/datadog-plugin-cucumber/src/index.js | 36 ++++- packages/datadog-plugin-cypress/src/plugin.js | 68 +++++++-- packages/datadog-plugin-jest/src/index.js | 42 +++++- packages/datadog-plugin-mocha/src/index.js | 33 ++++- .../datadog-plugin-playwright/src/index.js | 18 ++- .../exporters/agentless/coverage-writer.js | 31 ++++- .../exporters/agentless/writer.js | 31 ++++- .../exporters/git/git_metadata.js | 40 +++++- .../get-itr-configuration.js | 19 ++- .../get-skippable-suites.js | 27 +++- .../dd-trace/src/ci-visibility/telemetry.js | 130 ++++++++++++++++++ .../src/encode/agentless-ci-visibility.js | 15 +- .../src/encode/coverage-ci-visibility.js | 14 ++ .../exporters/common/agent-info-exporter.js | 4 + .../src/exporters/common/form-data.js | 4 + packages/dd-trace/src/plugins/ci_plugin.js | 52 +++++-- packages/dd-trace/src/plugins/util/exec.js | 25 +++- packages/dd-trace/src/plugins/util/git.js | 113 ++++++++++++--- packages/dd-trace/src/telemetry/index.js | 4 + .../agentless/coverage-writer.spec.js | 9 +- .../exporters/ci-visibility-exporter.spec.js | 2 + .../dd-trace/test/plugins/util/git.spec.js | 63 ++++----- 23 files changed, 688 insertions(+), 94 deletions(-) create mode 100644 packages/dd-trace/src/ci-visibility/telemetry.js diff --git a/packages/datadog-instrumentations/src/jest.js b/packages/datadog-instrumentations/src/jest.js index f62f0c9fac9..82935351e7d 100644 --- a/packages/datadog-instrumentations/src/jest.js +++ b/packages/datadog-instrumentations/src/jest.js @@ -403,7 +403,7 @@ function jestAdapterWrapper (jestAdapter, jestVersion) { const coverageFiles = getCoveredFilenamesFromCoverage(environment.global.__coverage__) .map(filename => getTestSuitePath(filename, environment.rootDir)) asyncResource.runInAsyncScope(() => { - testSuiteCodeCoverageCh.publish([...coverageFiles, environment.testSuite]) + testSuiteCodeCoverageCh.publish({ coverageFiles, testSuite: environment.testSuite }) }) } testSuiteFinishCh.publish({ status, errorMessage }) diff --git a/packages/datadog-plugin-cucumber/src/index.js b/packages/datadog-plugin-cucumber/src/index.js index 98fa1b4037c..4f2d29603b7 100644 --- a/packages/datadog-plugin-cucumber/src/index.js +++ b/packages/datadog-plugin-cucumber/src/index.js @@ -12,10 +12,21 @@ const { getTestSuiteCommonTags, addIntelligentTestRunnerSpanTags, TEST_ITR_UNSKIPPABLE, - TEST_ITR_FORCED_RUN + TEST_ITR_FORCED_RUN, + TEST_CODE_OWNERS } = require('../../dd-trace/src/plugins/util/test') const { RESOURCE_NAME } = require('../../../ext/tags') const { COMPONENT, ERROR_MESSAGE } = require('../../dd-trace/src/constants') +const { + TELEMETRY_EVENT_CREATED, + TELEMETRY_EVENT_FINISHED, + TELEMETRY_CODE_COVERAGE_STARTED, + TELEMETRY_CODE_COVERAGE_FINISHED, + TELEMETRY_ITR_FORCED_TO_RUN, + TELEMETRY_CODE_COVERAGE_EMPTY, + TELEMETRY_ITR_UNSKIPPABLE, + TELEMETRY_CODE_COVERAGE_NUM_FILES +} = require('../../dd-trace/src/ci-visibility/telemetry') class CucumberPlugin extends CiPlugin { static get id () { @@ -54,7 +65,9 @@ class CucumberPlugin extends CiPlugin { this.testSessionSpan.setTag(TEST_STATUS, status) this.testModuleSpan.setTag(TEST_STATUS, status) this.testModuleSpan.finish() + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module') this.testSessionSpan.finish() + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'session') finishAllTraceSpans(this.testSessionSpan) this.itrConfig = null @@ -69,9 +82,11 @@ class CucumberPlugin extends CiPlugin { 'cucumber' ) if (isUnskippable) { + this.telemetry.count(TELEMETRY_ITR_UNSKIPPABLE, { testLevel: 'suite' }) testSuiteMetadata[TEST_ITR_UNSKIPPABLE] = 'true' } if (isForcedToRun) { + this.telemetry.count(TELEMETRY_ITR_FORCED_TO_RUN, { testLevel: 'suite' }) testSuiteMetadata[TEST_ITR_FORCED_RUN] = 'true' } this.testSuiteSpan = this.tracer.startSpan('cucumber.test_suite', { @@ -82,20 +97,31 @@ class CucumberPlugin extends CiPlugin { ...testSuiteMetadata } }) + this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'suite') + if (this.itrConfig?.isCodeCoverageEnabled) { + this.telemetry.ciVisEvent(TELEMETRY_CODE_COVERAGE_STARTED, 'suite', { library: 'istanbul' }) + } }) this.addSub('ci:cucumber:test-suite:finish', status => { this.testSuiteSpan.setTag(TEST_STATUS, status) this.testSuiteSpan.finish() + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'suite') }) this.addSub('ci:cucumber:test-suite:code-coverage', ({ coverageFiles, suiteFile }) => { - if (!this.itrConfig || !this.itrConfig.isCodeCoverageEnabled) { + if (!this.itrConfig?.isCodeCoverageEnabled) { return } + if (!coverageFiles.length) { + this.telemetry.count(TELEMETRY_CODE_COVERAGE_EMPTY) + } + const relativeCoverageFiles = [...coverageFiles, suiteFile] .map(filename => getTestSuitePath(filename, this.sourceRoot)) + this.telemetry.distribution(TELEMETRY_CODE_COVERAGE_NUM_FILES, {}, relativeCoverageFiles.length) + const formattedCoverage = { sessionId: this.testSuiteSpan.context()._traceId, suiteId: this.testSuiteSpan.context()._spanId, @@ -103,6 +129,7 @@ class CucumberPlugin extends CiPlugin { } this.tracer._exporter.exportCoverage(formattedCoverage) + this.telemetry.ciVisEvent(TELEMETRY_CODE_COVERAGE_FINISHED, 'suite', { library: 'istanbul' }) }) this.addSub('ci:cucumber:test:start', ({ testName, fullTestSuite, testSourceLine }) => { @@ -142,6 +169,11 @@ class CucumberPlugin extends CiPlugin { } span.finish() + this.telemetry.ciVisEvent( + TELEMETRY_EVENT_FINISHED, + 'test', + { hasCodeOwners: !!span.context()._tags[TEST_CODE_OWNERS] } + ) if (!isStep) { finishAllTraceSpans(span) } diff --git a/packages/datadog-plugin-cypress/src/plugin.js b/packages/datadog-plugin-cypress/src/plugin.js index 6b767501362..abd6abec33b 100644 --- a/packages/datadog-plugin-cypress/src/plugin.js +++ b/packages/datadog-plugin-cypress/src/plugin.js @@ -29,6 +29,29 @@ const { ORIGIN_KEY, COMPONENT } = require('../../dd-trace/src/constants') const log = require('../../dd-trace/src/log') const NoopTracer = require('../../dd-trace/src/noop/tracer') const { isMarkedAsUnskippable } = require('../../datadog-plugin-jest/src/util') +const { + TELEMETRY_EVENT_CREATED, + TELEMETRY_EVENT_FINISHED, + TELEMETRY_ITR_FORCED_TO_RUN, + TELEMETRY_CODE_COVERAGE_EMPTY, + TELEMETRY_ITR_UNSKIPPABLE, + TELEMETRY_CODE_COVERAGE_NUM_FILES, + incrementCountMetric, + distributionMetric +} = require('../../dd-trace/src/ci-visibility/telemetry') +const { + GIT_REPOSITORY_URL, + GIT_COMMIT_SHA, + GIT_BRANCH, + CI_PROVIDER_NAME +} = require('../../dd-trace/src/plugins/util/tags') +const { + OS_VERSION, + OS_PLATFORM, + OS_ARCHITECTURE, + RUNTIME_NAME, + RUNTIME_VERSION +} = require('../../dd-trace/src/plugins/util/env') const TEST_FRAMEWORK_NAME = 'cypress' @@ -152,16 +175,19 @@ module.exports = (on, config) => { const testEnvironmentMetadata = getTestEnvironmentMetadata(TEST_FRAMEWORK_NAME) const { - 'git.repository_url': repositoryUrl, - 'git.commit.sha': sha, - 'os.version': osVersion, - 'os.platform': osPlatform, - 'os.architecture': osArchitecture, - 'runtime.name': runtimeName, - 'runtime.version': runtimeVersion, - 'git.branch': branch + [GIT_REPOSITORY_URL]: repositoryUrl, + [GIT_COMMIT_SHA]: sha, + [OS_VERSION]: osVersion, + [OS_PLATFORM]: osPlatform, + [OS_ARCHITECTURE]: osArchitecture, + [RUNTIME_NAME]: runtimeName, + [RUNTIME_VERSION]: runtimeVersion, + [GIT_BRANCH]: branch, + [CI_PROVIDER_NAME]: ciProviderName } = testEnvironmentMetadata + const isUnsupportedCIProvider = !ciProviderName + const finishedTestsByFile = {} const testConfiguration = { @@ -192,6 +218,15 @@ module.exports = (on, config) => { let hasForcedToRunSuites = false let hasUnskippableSuites = false + function ciVisEvent (name, testLevel, tags = {}) { + incrementCountMetric(name, { + testLevel, + testFramework: 'cypress', + isUnsupportedCIProvider, + ...tags + }) + } + function getTestSpan (testName, testSuite, isUnskippable, isForcedToRun) { const testSuiteTags = { [TEST_COMMAND]: command, @@ -220,14 +255,18 @@ module.exports = (on, config) => { if (isUnskippable) { hasUnskippableSuites = true + incrementCountMetric(TELEMETRY_ITR_UNSKIPPABLE, { testLevel: 'suite' }) testSpanMetadata[TEST_ITR_UNSKIPPABLE] = 'true' } if (isForcedToRun) { hasForcedToRunSuites = true + incrementCountMetric(TELEMETRY_ITR_FORCED_TO_RUN, { testLevel: 'suite' }) testSpanMetadata[TEST_ITR_FORCED_RUN] = 'true' } + ciVisEvent(TELEMETRY_EVENT_CREATED, 'test', { hasCodeOwners: !!codeOwners }) + return tracer.startSpan(`${TEST_FRAMEWORK_NAME}.test`, { childOf, tags: { @@ -281,6 +320,8 @@ module.exports = (on, config) => { ...testSessionSpanMetadata } }) + ciVisEvent(TELEMETRY_EVENT_CREATED, 'session') + testModuleSpan = tracer.startSpan(`${TEST_FRAMEWORK_NAME}.test_module`, { childOf: testSessionSpan, tags: { @@ -289,6 +330,8 @@ module.exports = (on, config) => { ...testModuleSpanMetadata } }) + ciVisEvent(TELEMETRY_EVENT_CREATED, 'module') + return details }) }) @@ -347,6 +390,7 @@ module.exports = (on, config) => { } testSuiteSpan.finish() testSuiteSpan = null + ciVisEvent(TELEMETRY_EVENT_FINISHED, 'suite') } }) @@ -371,7 +415,9 @@ module.exports = (on, config) => { ) testModuleSpan.finish() + ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module') testSessionSpan.finish() + ciVisEvent(TELEMETRY_EVENT_FINISHED, 'session') finishAllTraceSpans(testSessionSpan) } @@ -406,6 +452,7 @@ module.exports = (on, config) => { ...testSuiteSpanMetadata } }) + ciVisEvent(TELEMETRY_EVENT_CREATED, 'suite') return null }, 'dd:beforeEach': (test) => { @@ -435,6 +482,10 @@ module.exports = (on, config) => { if (coverage && isCodeCoverageEnabled && tracer._tracer._exporter && tracer._tracer._exporter.exportCoverage) { const coverageFiles = getCoveredFilenamesFromCoverage(coverage) const relativeCoverageFiles = coverageFiles.map(file => getTestSuitePath(file, rootDir)) + if (!relativeCoverageFiles.length) { + incrementCountMetric(TELEMETRY_CODE_COVERAGE_EMPTY) + } + distributionMetric(TELEMETRY_CODE_COVERAGE_NUM_FILES, {}, relativeCoverageFiles.length) const { _traceId, _spanId } = testSuiteSpan.context() const formattedCoverage = { sessionId: _traceId, @@ -470,6 +521,7 @@ module.exports = (on, config) => { // test spans are finished at after:spec } activeSpan = null + ciVisEvent(TELEMETRY_EVENT_FINISHED, 'test') return null }, 'dd:addTags': (tags) => { diff --git a/packages/datadog-plugin-jest/src/index.js b/packages/datadog-plugin-jest/src/index.js index 3eaceb034aa..6c4fed9755e 100644 --- a/packages/datadog-plugin-jest/src/index.js +++ b/packages/datadog-plugin-jest/src/index.js @@ -12,10 +12,21 @@ const { TEST_FRAMEWORK_VERSION, TEST_SOURCE_START, TEST_ITR_UNSKIPPABLE, - TEST_ITR_FORCED_RUN + TEST_ITR_FORCED_RUN, + TEST_CODE_OWNERS } = require('../../dd-trace/src/plugins/util/test') const { COMPONENT } = require('../../dd-trace/src/constants') const id = require('../../dd-trace/src/id') +const { + TELEMETRY_EVENT_CREATED, + TELEMETRY_EVENT_FINISHED, + TELEMETRY_CODE_COVERAGE_STARTED, + TELEMETRY_CODE_COVERAGE_FINISHED, + TELEMETRY_ITR_FORCED_TO_RUN, + TELEMETRY_CODE_COVERAGE_EMPTY, + TELEMETRY_ITR_UNSKIPPABLE, + TELEMETRY_CODE_COVERAGE_NUM_FILES +} = require('../../dd-trace/src/ci-visibility/telemetry') const isJestWorker = !!process.env.JEST_WORKER_ID @@ -81,7 +92,9 @@ class JestPlugin extends CiPlugin { ) this.testModuleSpan.finish() + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module') this.testSessionSpan.finish() + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'session') finishAllTraceSpans(this.testSessionSpan) this.tracer._exporter.flush() }) @@ -103,7 +116,8 @@ class JestPlugin extends CiPlugin { _ddTestCommand: testCommand, _ddTestModuleId: testModuleId, _ddForcedToRun, - _ddUnskippable + _ddUnskippable, + _ddTestCodeCoverageEnabled } = testEnvironmentOptions const testSessionSpanContext = this.tracer.extract('text_map', { @@ -114,8 +128,10 @@ class JestPlugin extends CiPlugin { const testSuiteMetadata = getTestSuiteCommonTags(testCommand, frameworkVersion, testSuite, 'jest') if (_ddUnskippable) { + this.telemetry.count(TELEMETRY_ITR_UNSKIPPABLE, { testLevel: 'suite' }) testSuiteMetadata[TEST_ITR_UNSKIPPABLE] = 'true' if (_ddForcedToRun) { + this.telemetry.count(TELEMETRY_ITR_FORCED_TO_RUN, { testLevel: 'suite' }) testSuiteMetadata[TEST_ITR_FORCED_RUN] = 'true' } } @@ -128,6 +144,10 @@ class JestPlugin extends CiPlugin { ...testSuiteMetadata } }) + this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'suite') + if (_ddTestCodeCoverageEnabled) { + this.telemetry.ciVisEvent(TELEMETRY_CODE_COVERAGE_STARTED, 'suite', { library: 'istanbul' }) + } }) this.addSub('ci:jest:worker-report:trace', traces => { @@ -164,6 +184,7 @@ class JestPlugin extends CiPlugin { this.testSuiteSpan.setTag('error', new Error(errorMessage)) } this.testSuiteSpan.finish() + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'suite') // Suites potentially run in a different process than the session, // so calling finishAllTraceSpans on the session span is not enough finishAllTraceSpans(this.testSuiteSpan) @@ -180,14 +201,22 @@ class JestPlugin extends CiPlugin { * because this subscription happens in a different process from the one * fetching the ITR config. */ - this.addSub('ci:jest:test-suite:code-coverage', (coverageFiles) => { + this.addSub('ci:jest:test-suite:code-coverage', ({ coverageFiles, testSuite }) => { + if (!coverageFiles.length) { + this.telemetry.count(TELEMETRY_CODE_COVERAGE_EMPTY) + } + const files = [...coverageFiles, testSuite] + const { _traceId, _spanId } = this.testSuiteSpan.context() const formattedCoverage = { sessionId: _traceId, suiteId: _spanId, - files: coverageFiles + files } + this.tracer._exporter.exportCoverage(formattedCoverage) + this.telemetry.ciVisEvent(TELEMETRY_CODE_COVERAGE_FINISHED, 'suite', { library: 'istanbul' }) + this.telemetry.distribution(TELEMETRY_CODE_COVERAGE_NUM_FILES, {}, files.length) }) this.addSub('ci:jest:test:start', (test) => { @@ -204,6 +233,11 @@ class JestPlugin extends CiPlugin { span.setTag(TEST_SOURCE_START, testStartLine) } span.finish() + this.telemetry.ciVisEvent( + TELEMETRY_EVENT_FINISHED, + 'test', + { hasCodeOwners: !!span.context()._tags[TEST_CODE_OWNERS] } + ) finishAllTraceSpans(span) }) diff --git a/packages/datadog-plugin-mocha/src/index.js b/packages/datadog-plugin-mocha/src/index.js index c8af76247b1..bef92ffe18e 100644 --- a/packages/datadog-plugin-mocha/src/index.js +++ b/packages/datadog-plugin-mocha/src/index.js @@ -13,9 +13,20 @@ const { addIntelligentTestRunnerSpanTags, TEST_SOURCE_START, TEST_ITR_UNSKIPPABLE, - TEST_ITR_FORCED_RUN + TEST_ITR_FORCED_RUN, + TEST_CODE_OWNERS } = require('../../dd-trace/src/plugins/util/test') const { COMPONENT } = require('../../dd-trace/src/constants') +const { + TELEMETRY_EVENT_CREATED, + TELEMETRY_EVENT_FINISHED, + TELEMETRY_CODE_COVERAGE_STARTED, + TELEMETRY_CODE_COVERAGE_FINISHED, + TELEMETRY_ITR_FORCED_TO_RUN, + TELEMETRY_CODE_COVERAGE_EMPTY, + TELEMETRY_ITR_UNSKIPPABLE, + TELEMETRY_CODE_COVERAGE_NUM_FILES +} = require('../../dd-trace/src/ci-visibility/telemetry') class MochaPlugin extends CiPlugin { static get id () { @@ -35,6 +46,10 @@ class MochaPlugin extends CiPlugin { } const testSuiteSpan = this._testSuites.get(suiteFile) + if (!coverageFiles.length) { + this.telemetry.count(TELEMETRY_CODE_COVERAGE_EMPTY) + } + const relativeCoverageFiles = [...coverageFiles, suiteFile] .map(filename => getTestSuitePath(filename, this.sourceRoot)) @@ -47,6 +62,8 @@ class MochaPlugin extends CiPlugin { } this.tracer._exporter.exportCoverage(formattedCoverage) + this.telemetry.ciVisEvent(TELEMETRY_CODE_COVERAGE_FINISHED, 'suite', { library: 'istanbul' }) + this.telemetry.distribution(TELEMETRY_CODE_COVERAGE_NUM_FILES, {}, relativeCoverageFiles.length) }) this.addSub('ci:mocha:test-suite:start', ({ testSuite, isUnskippable, isForcedToRun }) => { @@ -59,9 +76,11 @@ class MochaPlugin extends CiPlugin { ) if (isUnskippable) { testSuiteMetadata[TEST_ITR_UNSKIPPABLE] = 'true' + this.telemetry.count(TELEMETRY_ITR_UNSKIPPABLE, { testLevel: 'suite' }) } if (isForcedToRun) { testSuiteMetadata[TEST_ITR_FORCED_RUN] = 'true' + this.telemetry.count(TELEMETRY_ITR_FORCED_TO_RUN, { testLevel: 'suite' }) } const testSuiteSpan = this.tracer.startSpan('mocha.test_suite', { @@ -72,6 +91,10 @@ class MochaPlugin extends CiPlugin { ...testSuiteMetadata } }) + this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'suite') + if (this.itrConfig?.isCodeCoverageEnabled) { + this.telemetry.ciVisEvent(TELEMETRY_CODE_COVERAGE_STARTED, 'suite', { library: 'istanbul' }) + } this.enter(testSuiteSpan, store) this._testSuites.set(testSuite, testSuiteSpan) }) @@ -85,6 +108,7 @@ class MochaPlugin extends CiPlugin { span.setTag(TEST_STATUS, status) } span.finish() + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'suite') } }) @@ -113,6 +137,11 @@ class MochaPlugin extends CiPlugin { span.setTag(TEST_STATUS, status) span.finish() + this.telemetry.ciVisEvent( + TELEMETRY_EVENT_FINISHED, + 'test', + { hasCodeOwners: !!span.context()._tags[TEST_CODE_OWNERS] } + ) finishAllTraceSpans(span) } }) @@ -179,7 +208,9 @@ class MochaPlugin extends CiPlugin { ) this.testModuleSpan.finish() + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module') this.testSessionSpan.finish() + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'session') finishAllTraceSpans(this.testSessionSpan) } this.itrConfig = null diff --git a/packages/datadog-plugin-playwright/src/index.js b/packages/datadog-plugin-playwright/src/index.js index 928477ffc3b..eb8810967c3 100644 --- a/packages/datadog-plugin-playwright/src/index.js +++ b/packages/datadog-plugin-playwright/src/index.js @@ -8,10 +8,15 @@ const { finishAllTraceSpans, getTestSuitePath, getTestSuiteCommonTags, - TEST_SOURCE_START + TEST_SOURCE_START, + TEST_CODE_OWNERS } = require('../../dd-trace/src/plugins/util/test') const { RESOURCE_NAME } = require('../../../ext/tags') const { COMPONENT } = require('../../dd-trace/src/constants') +const { + TELEMETRY_EVENT_CREATED, + TELEMETRY_EVENT_FINISHED +} = require('../../dd-trace/src/ci-visibility/telemetry') class PlaywrightPlugin extends CiPlugin { static get id () { @@ -28,7 +33,9 @@ class PlaywrightPlugin extends CiPlugin { this.testSessionSpan.setTag(TEST_STATUS, status) this.testModuleSpan.finish() + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module') this.testSessionSpan.finish() + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'session') finishAllTraceSpans(this.testSessionSpan) this.tracer._exporter.flush(onDone) }) @@ -52,6 +59,7 @@ class PlaywrightPlugin extends CiPlugin { ...testSuiteMetadata } }) + this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'suite') this.enter(testSuiteSpan, store) this._testSuites.set(testSuite, testSuiteSpan) @@ -63,6 +71,7 @@ class PlaywrightPlugin extends CiPlugin { if (!span) return span.setTag(TEST_STATUS, status) span.finish() + this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'suite') }) this.addSub('ci:playwright:test:start', ({ testName, testSuiteAbsolutePath, testSourceLine }) => { @@ -104,6 +113,13 @@ class PlaywrightPlugin extends CiPlugin { }) span.finish() + + this.telemetry.ciVisEvent( + TELEMETRY_EVENT_FINISHED, + 'test', + { hasCodeOwners: !!span.context()._tags[TEST_CODE_OWNERS] } + ) + finishAllTraceSpans(span) }) } diff --git a/packages/dd-trace/src/ci-visibility/exporters/agentless/coverage-writer.js b/packages/dd-trace/src/ci-visibility/exporters/agentless/coverage-writer.js index 8728e4a2e04..52001672101 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/agentless/coverage-writer.js +++ b/packages/dd-trace/src/ci-visibility/exporters/agentless/coverage-writer.js @@ -5,6 +5,16 @@ const { safeJSONStringify } = require('../../../exporters/common/util') const { CoverageCIVisibilityEncoder } = require('../../../encode/coverage-ci-visibility') const BaseWriter = require('../../../exporters/common/writer') +const { + incrementCountMetric, + distributionMetric, + TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS, + TELEMETRY_ENDPOINT_PAYLOAD_BYTES, + TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_MS, + TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS, + TELEMETRY_ENDPOINT_PAYLOAD_DROPPED, + getErrorTypeFromStatusCode +} = require('../../../ci-visibility/telemetry') class Writer extends BaseWriter { constructor ({ url, evpProxyPrefix = '' }) { @@ -34,8 +44,27 @@ class Writer extends BaseWriter { log.debug(() => `Request to the intake: ${safeJSONStringify(options)}`) - request(form, options, (err, res) => { + const startRequestTime = Date.now() + + incrementCountMetric(TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS, { endpoint: 'code_coverage' }) + distributionMetric(TELEMETRY_ENDPOINT_PAYLOAD_BYTES, { endpoint: 'code_coverage' }, form.size()) + + request(form, options, (err, res, statusCode) => { + distributionMetric( + TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_MS, + { endpoint: 'code_coverage' }, + Date.now() - startRequestTime + ) if (err) { + const errorType = getErrorTypeFromStatusCode(statusCode) + incrementCountMetric( + TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS, + { endpoint: 'code_coverage', errorType } + ) + incrementCountMetric( + TELEMETRY_ENDPOINT_PAYLOAD_DROPPED, + { endpoint: 'code_coverage' } + ) log.error(err) done() return diff --git a/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js b/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js index d04406f33b9..afbc670443e 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js +++ b/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js @@ -5,6 +5,16 @@ const log = require('../../../log') const { AgentlessCiVisibilityEncoder } = require('../../../encode/agentless-ci-visibility') const BaseWriter = require('../../../exporters/common/writer') +const { + incrementCountMetric, + distributionMetric, + TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS, + TELEMETRY_ENDPOINT_PAYLOAD_BYTES, + TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_MS, + TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS, + TELEMETRY_ENDPOINT_PAYLOAD_DROPPED, + getErrorTypeFromStatusCode +} = require('../../../ci-visibility/telemetry') class Writer extends BaseWriter { constructor ({ url, tags, evpProxyPrefix = '' }) { @@ -35,8 +45,27 @@ class Writer extends BaseWriter { log.debug(() => `Request to the intake: ${safeJSONStringify(options)}`) - request(data, options, (err, res) => { + const startRequestTime = Date.now() + + incrementCountMetric(TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS, { endpoint: 'test_cycle' }) + distributionMetric(TELEMETRY_ENDPOINT_PAYLOAD_BYTES, { endpoint: 'test_cycle' }, data.length) + + request(data, options, (err, res, statusCode) => { + distributionMetric( + TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_MS, + { endpoint: 'test_cycle' }, + Date.now() - startRequestTime + ) if (err) { + const errorType = getErrorTypeFromStatusCode(statusCode) + incrementCountMetric( + TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS, + { endpoint: 'test_cycle', errorType } + ) + incrementCountMetric( + TELEMETRY_ENDPOINT_PAYLOAD_DROPPED, + { endpoint: 'test_cycle' } + ) log.error(err) done() return diff --git a/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js b/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js index 2747b9746e7..fb0329ab637 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js +++ b/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js @@ -15,6 +15,20 @@ const { unshallowRepository } = require('../../../plugins/util/git') +const { + incrementCountMetric, + distributionMetric, + TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS, + TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_MS, + TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_ERRORS, + TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_NUM, + TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES, + TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_MS, + TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_ERRORS, + TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_BYTES, + getErrorTypeFromStatusCode +} = require('../../../ci-visibility/telemetry') + const isValidSha1 = (sha) => /^[0-9a-f]{40}$/.test(sha) const isValidSha256 = (sha) => /^[0-9a-f]{64}$/.test(sha) @@ -74,8 +88,13 @@ function getCommitsToUpload ({ url, repositoryUrl, latestCommits, isEvpProxy }, })) }) - request(localCommitData, options, (err, response) => { + incrementCountMetric(TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS) + const startTime = Date.now() + request(localCommitData, options, (err, response, statusCode) => { + distributionMetric(TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_MS, {}, Date.now() - startTime) if (err) { + const errorType = getErrorTypeFromStatusCode(statusCode) + incrementCountMetric(TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_ERRORS, { errorType }) const error = new Error(`Error fetching commits to exclude: ${err.message}`) return callback(error) } @@ -83,6 +102,7 @@ function getCommitsToUpload ({ url, repositoryUrl, latestCommits, isEvpProxy }, try { alreadySeenCommits = validateCommits(JSON.parse(response).data) } catch (e) { + incrementCountMetric(TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_ERRORS, { errorType: 'network' }) return callback(new Error(`Can't parse commits to exclude response: ${e.message}`)) } log.debug(`There are ${alreadySeenCommits.length} commits to exclude.`) @@ -147,12 +167,20 @@ function uploadPackFile ({ url, isEvpProxy, packFileToUpload, repositoryUrl, hea delete options.headers['dd-api-key'] } + incrementCountMetric(TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES) + + const uploadSize = form.size() + + const startTime = Date.now() request(form, options, (err, _, statusCode) => { + distributionMetric(TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_MS, {}, Date.now() - startTime) if (err) { + const errorType = getErrorTypeFromStatusCode(statusCode) + incrementCountMetric(TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_ERRORS, { errorType }) const error = new Error(`Could not upload packfiles: status code ${statusCode}: ${err.message}`) - return callback(error) + return callback(error, uploadSize) } - callback(null) + callback(null, uploadSize) }) } @@ -173,10 +201,14 @@ function generateAndUploadPackFiles ({ return callback(new Error('Failed to generate packfiles')) } + distributionMetric(TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_NUM, {}, packFilesToUpload.length) let packFileIndex = 0 + let totalUploadedBytes = 0 // This uploads packfiles sequentially - const uploadPackFileCallback = (err) => { + const uploadPackFileCallback = (err, byteLength) => { + totalUploadedBytes += byteLength if (err || packFileIndex === packFilesToUpload.length) { + distributionMetric(TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_BYTES, {}, totalUploadedBytes) return callback(err) } return uploadPackFile( diff --git a/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-itr-configuration.js b/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-itr-configuration.js index 2aee819004d..6df4d99ea98 100644 --- a/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-itr-configuration.js +++ b/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-itr-configuration.js @@ -1,6 +1,15 @@ const request = require('../../exporters/common/request') const id = require('../../id') const log = require('../../log') +const { + incrementCountMetric, + distributionMetric, + TELEMETRY_GIT_REQUESTS_SETTINGS, + TELEMETRY_GIT_REQUESTS_SETTINGS_MS, + TELEMETRY_GIT_REQUESTS_SETTINGS_ERRORS, + TELEMETRY_GIT_REQUESTS_SETTINGS_RESPONSE, + getErrorTypeFromStatusCode +} = require('../../ci-visibility/telemetry') function getItrConfiguration ({ url, @@ -62,8 +71,14 @@ function getItrConfiguration ({ } }) - request(data, options, (err, res) => { + incrementCountMetric(TELEMETRY_GIT_REQUESTS_SETTINGS) + + const startTime = Date.now() + request(data, options, (err, res, statusCode) => { + distributionMetric(TELEMETRY_GIT_REQUESTS_SETTINGS_MS, {}, Date.now() - startTime) if (err) { + const errorType = getErrorTypeFromStatusCode(statusCode) + incrementCountMetric(TELEMETRY_GIT_REQUESTS_SETTINGS_ERRORS, { errorType }) done(err) } else { try { @@ -91,6 +106,8 @@ function getItrConfiguration ({ log.debug(() => 'Dangerously set test skipping to true') } + incrementCountMetric(TELEMETRY_GIT_REQUESTS_SETTINGS_RESPONSE, settings) + done(null, settings) } catch (err) { done(err) diff --git a/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-skippable-suites.js b/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-skippable-suites.js index 04448e9a651..7ee0091a7cb 100644 --- a/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-skippable-suites.js +++ b/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-skippable-suites.js @@ -1,5 +1,16 @@ const request = require('../../exporters/common/request') const log = require('../../log') +const { + incrementCountMetric, + distributionMetric, + TELEMETRY_ITR_SKIPPABLE_TESTS, + TELEMETRY_ITR_SKIPPABLE_TESTS_MS, + TELEMETRY_ITR_SKIPPABLE_TESTS_ERRORS, + TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_SUITES, + TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_TESTS, + TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_BYTES, + getErrorTypeFromStatusCode +} = require('../../ci-visibility/telemetry') function getSkippableSuites ({ url, @@ -59,8 +70,15 @@ function getSkippableSuites ({ } }) - request(data, options, (err, res) => { + incrementCountMetric(TELEMETRY_ITR_SKIPPABLE_TESTS) + + const startTime = Date.now() + + request(data, options, (err, res, statusCode) => { + distributionMetric(TELEMETRY_ITR_SKIPPABLE_TESTS_MS, {}, Date.now() - startTime) if (err) { + const errorType = getErrorTypeFromStatusCode(statusCode) + incrementCountMetric(TELEMETRY_ITR_SKIPPABLE_TESTS_ERRORS, { errorType }) done(err) } else { let skippableSuites = [] @@ -74,6 +92,13 @@ function getSkippableSuites ({ } return { suite, name } }) + incrementCountMetric( + testLevel === 'test' + ? TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_TESTS : TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_SUITES, + {}, + skippableSuites.length + ) + distributionMetric(TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_BYTES, {}, res.length) log.debug(() => `Number of received skippable ${testLevel}s: ${skippableSuites.length}`) done(null, skippableSuites) } catch (err) { diff --git a/packages/dd-trace/src/ci-visibility/telemetry.js b/packages/dd-trace/src/ci-visibility/telemetry.js new file mode 100644 index 00000000000..1bc01c502c9 --- /dev/null +++ b/packages/dd-trace/src/ci-visibility/telemetry.js @@ -0,0 +1,130 @@ +const telemetryMetrics = require('../telemetry/metrics') + +const ciVisibilityMetrics = telemetryMetrics.manager.namespace('civisibility') + +const formattedTags = { + testLevel: 'event_type', + testFramework: 'test_framework', + errorType: 'error_type', + exitCode: 'exit_code', + isCodeCoverageEnabled: 'coverage_enabled', + isSuitesSkippingEnabled: 'itrskip_enabled', + hasCodeOwners: 'has_code_owners', + isUnsupportedCIProvider: 'is_unsupported_ci' +} + +// Transform tags dictionary to array of strings. +// If tag value is true, then only tag key is added to the array. +function formatMetricTags (tagsDictionary) { + return Object.keys(tagsDictionary).reduce((acc, tagKey) => { + const formattedTagKey = formattedTags[tagKey] || tagKey + if (tagsDictionary[tagKey] === true) { + acc.push(formattedTagKey) + } else if (tagsDictionary[tagKey] !== undefined && tagsDictionary[tagKey] !== null) { + acc.push(`${formattedTagKey}:${tagsDictionary[tagKey]}`) + } + return acc + }, []) +} + +function incrementCountMetric (name, tags = {}, value = 1) { + ciVisibilityMetrics.count(name, formatMetricTags(tags)).inc(value) +} + +function distributionMetric (name, tags, measure) { + ciVisibilityMetrics.distribution(name, formatMetricTags(tags)).track(measure) +} + +// CI Visibility telemetry events +const TELEMETRY_EVENT_CREATED = 'event_created' +const TELEMETRY_EVENT_FINISHED = 'event_finished' +const TELEMETRY_CODE_COVERAGE_STARTED = 'code_coverage_started' +const TELEMETRY_CODE_COVERAGE_FINISHED = 'code_coverage_finished' +const TELEMETRY_ITR_SKIPPED = 'itr_skipped' +const TELEMETRY_ITR_UNSKIPPABLE = 'itr_unskippable' +const TELEMETRY_ITR_FORCED_TO_RUN = 'itr_forced_run' +const TELEMETRY_CODE_COVERAGE_EMPTY = 'code_coverage.is_empty' +const TELEMETRY_CODE_COVERAGE_NUM_FILES = 'code_coverage.files' +const TELEMETRY_EVENTS_ENQUEUED_FOR_SERIALIZATION = 'events_enqueued_for_serialization' +const TELEMETRY_ENDPOINT_PAYLOAD_SERIALIZATION_MS = 'endpoint_payload.events_serialization_ms' +const TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS = 'endpoint_payload.requests' +const TELEMETRY_ENDPOINT_PAYLOAD_BYTES = 'endpoint_payload.bytes' +const TELEMETRY_ENDPOINT_PAYLOAD_EVENTS_COUNT = 'endpoint_payload.events_count' +const TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_MS = 'endpoint_payload.requests_ms' +const TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS = 'endpoint_payload.requests_errors' +const TELEMETRY_ENDPOINT_PAYLOAD_DROPPED = 'endpoint_payload.dropped' +const TELEMETRY_GIT_COMMAND = 'git.command' +const TELEMETRY_GIT_COMMAND_MS = 'git.command_ms' +const TELEMETRY_GIT_COMMAND_ERRORS = 'git.command_errors' +const TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS = 'git_requests.search_commits' +const TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_MS = 'git_requests.search_commits_ms' +const TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_ERRORS = 'git_requests.search_commits_errors' +const TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES = 'git_requests.objects_pack' +const TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_MS = 'git_requests.objects_pack_ms' +const TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_ERRORS = 'git_requests.objects_pack_errors' +const TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_NUM = 'git_requests.objects_pack_files' +const TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_BYTES = 'git_requests.objects_pack_bytes' +const TELEMETRY_GIT_REQUESTS_SETTINGS = 'git_requests.settings' +const TELEMETRY_GIT_REQUESTS_SETTINGS_MS = 'git_requests.settings_ms' +const TELEMETRY_GIT_REQUESTS_SETTINGS_ERRORS = 'git_requests.settings_errors' +const TELEMETRY_GIT_REQUESTS_SETTINGS_RESPONSE = 'git_requests.settings_response' +const TELEMETRY_ITR_SKIPPABLE_TESTS = 'itr_skippable_tests.request' +const TELEMETRY_ITR_SKIPPABLE_TESTS_MS = 'itr_skippable_tests.request_ms' +const TELEMETRY_ITR_SKIPPABLE_TESTS_ERRORS = 'itr_skippable_tests.request_errors' +const TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_SUITES = 'itr_skippable_tests.response_suites' +const TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_TESTS = 'itr_skippable_tests.response_tests' +const TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_BYTES = 'itr_skippable_tests.response_bytes' + +function getErrorTypeFromStatusCode (statusCode) { + if (statusCode >= 400 && statusCode < 500) { + return 'status_code_4xx_response' + } + if (statusCode >= 500) { + return 'status_code_5xx_response' + } + return 'network' +} + +module.exports = { + incrementCountMetric, + distributionMetric, + TELEMETRY_EVENT_CREATED, + TELEMETRY_EVENT_FINISHED, + TELEMETRY_CODE_COVERAGE_STARTED, + TELEMETRY_CODE_COVERAGE_FINISHED, + TELEMETRY_ITR_SKIPPED, + TELEMETRY_ITR_UNSKIPPABLE, + TELEMETRY_ITR_FORCED_TO_RUN, + TELEMETRY_CODE_COVERAGE_EMPTY, + TELEMETRY_CODE_COVERAGE_NUM_FILES, + TELEMETRY_EVENTS_ENQUEUED_FOR_SERIALIZATION, + TELEMETRY_ENDPOINT_PAYLOAD_SERIALIZATION_MS, + TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS, + TELEMETRY_ENDPOINT_PAYLOAD_BYTES, + TELEMETRY_ENDPOINT_PAYLOAD_EVENTS_COUNT, + TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_MS, + TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS, + TELEMETRY_ENDPOINT_PAYLOAD_DROPPED, + TELEMETRY_GIT_COMMAND, + TELEMETRY_GIT_COMMAND_MS, + TELEMETRY_GIT_COMMAND_ERRORS, + TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS, + TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_MS, + TELEMETRY_GIT_REQUESTS_SEARCH_COMMITS_ERRORS, + TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_NUM, + TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_BYTES, + TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES, + TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_MS, + TELEMETRY_GIT_REQUESTS_OBJECT_PACKFILES_ERRORS, + TELEMETRY_GIT_REQUESTS_SETTINGS, + TELEMETRY_GIT_REQUESTS_SETTINGS_MS, + TELEMETRY_GIT_REQUESTS_SETTINGS_ERRORS, + TELEMETRY_GIT_REQUESTS_SETTINGS_RESPONSE, + TELEMETRY_ITR_SKIPPABLE_TESTS, + TELEMETRY_ITR_SKIPPABLE_TESTS_MS, + TELEMETRY_ITR_SKIPPABLE_TESTS_ERRORS, + TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_SUITES, + TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_TESTS, + TELEMETRY_ITR_SKIPPABLE_TESTS_RESPONSE_BYTES, + getErrorTypeFromStatusCode +} diff --git a/packages/dd-trace/src/encode/agentless-ci-visibility.js b/packages/dd-trace/src/encode/agentless-ci-visibility.js index c0111d19679..cf9c180ce3a 100644 --- a/packages/dd-trace/src/encode/agentless-ci-visibility.js +++ b/packages/dd-trace/src/encode/agentless-ci-visibility.js @@ -3,8 +3,13 @@ const { truncateSpan, normalizeSpan } = require('./tags-processors') const { AgentEncoder } = require('./0.4') const { version: ddTraceVersion } = require('../../../../package.json') const id = require('../../../dd-trace/src/id') -const ENCODING_VERSION = 1 +const { + distributionMetric, + TELEMETRY_ENDPOINT_PAYLOAD_SERIALIZATION_MS, + TELEMETRY_ENDPOINT_PAYLOAD_EVENTS_COUNT +} = require('../ci-visibility/telemetry') +const ENCODING_VERSION = 1 const ALLOWED_CONTENT_TYPES = ['test_session_end', 'test_module_end', 'test_suite_end', 'test'] const TEST_SUITE_KEYS_LENGTH = 12 @@ -247,6 +252,8 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder { } _encode (bytes, trace) { + const startTime = Date.now() + const rawEvents = trace.map(formatSpan) const testSessionEvents = rawEvents.filter( @@ -261,9 +268,15 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder { for (const event of events) { this._encodeEvent(bytes, event) } + distributionMetric( + TELEMETRY_ENDPOINT_PAYLOAD_SERIALIZATION_MS, + { endpoint: 'test_cycle' }, + Date.now() - startTime + ) } makePayload () { + distributionMetric(TELEMETRY_ENDPOINT_PAYLOAD_EVENTS_COUNT, { endpoint: 'test_cycle' }, this._eventCount) const bytes = this._traceBytes const eventsOffset = this._eventsOffset const eventsCount = this._eventCount diff --git a/packages/dd-trace/src/encode/coverage-ci-visibility.js b/packages/dd-trace/src/encode/coverage-ci-visibility.js index a877e11e864..75da679340c 100644 --- a/packages/dd-trace/src/encode/coverage-ci-visibility.js +++ b/packages/dd-trace/src/encode/coverage-ci-visibility.js @@ -2,6 +2,11 @@ const { AgentEncoder } = require('./0.4') const Chunk = require('./chunk') +const { + distributionMetric, + TELEMETRY_ENDPOINT_PAYLOAD_SERIALIZATION_MS, + TELEMETRY_ENDPOINT_PAYLOAD_EVENTS_COUNT +} = require('../ci-visibility/telemetry') const FormData = require('../exporters/common/form-data') const COVERAGE_PAYLOAD_VERSION = 2 @@ -21,8 +26,16 @@ class CoverageCIVisibilityEncoder extends AgentEncoder { } encode (coverage) { + const startTime = Date.now() + this._coveragesCount++ this.encodeCodeCoverage(this._coverageBytes, coverage) + + distributionMetric( + TELEMETRY_ENDPOINT_PAYLOAD_SERIALIZATION_MS, + { endpoint: 'code_coverage' }, + Date.now() - startTime + ) } encodeCodeCoverage (bytes, coverage) { @@ -73,6 +86,7 @@ class CoverageCIVisibilityEncoder extends AgentEncoder { } makePayload () { + distributionMetric(TELEMETRY_ENDPOINT_PAYLOAD_EVENTS_COUNT, { endpoint: 'code_coverage' }, this._coveragesCount) const bytes = this._coverageBytes const coveragesOffset = this._coveragesOffset diff --git a/packages/dd-trace/src/exporters/common/agent-info-exporter.js b/packages/dd-trace/src/exporters/common/agent-info-exporter.js index 9d1c45195bc..923b7eef0ef 100644 --- a/packages/dd-trace/src/exporters/common/agent-info-exporter.js +++ b/packages/dd-trace/src/exporters/common/agent-info-exporter.js @@ -1,6 +1,7 @@ const { URL, format } = require('url') const request = require('./request') +const { incrementCountMetric, TELEMETRY_EVENTS_ENQUEUED_FOR_SERIALIZATION } = require('../../ci-visibility/telemetry') function fetchAgentInfo (url, callback) { request('', { @@ -49,6 +50,9 @@ class AgentInfoExporter { } _export (payload, writer = this._writer, timerKey = '_timer') { + if (this._config.isCiVisibility) { + incrementCountMetric(TELEMETRY_EVENTS_ENQUEUED_FOR_SERIALIZATION, {}, payload.length) + } writer.append(payload) const { flushInterval } = this._config diff --git a/packages/dd-trace/src/exporters/common/form-data.js b/packages/dd-trace/src/exporters/common/form-data.js index b20e97b8864..dacd495b160 100644 --- a/packages/dd-trace/src/exporters/common/form-data.js +++ b/packages/dd-trace/src/exporters/common/form-data.js @@ -21,6 +21,10 @@ class FormData extends Readable { } } + size () { + return this._data.reduce((size, chunk) => size + chunk.length, 0) + } + getHeaders () { return { 'Content-Type': 'multipart/form-data; boundary=' + this._boundary } } diff --git a/packages/dd-trace/src/plugins/ci_plugin.js b/packages/dd-trace/src/plugins/ci_plugin.js index 0112c4cb4fa..5d9ff3af5cf 100644 --- a/packages/dd-trace/src/plugins/ci_plugin.js +++ b/packages/dd-trace/src/plugins/ci_plugin.js @@ -20,6 +20,14 @@ const { const Plugin = require('./plugin') const { COMPONENT } = require('../constants') const log = require('../log') +const { + incrementCountMetric, + distributionMetric, + TELEMETRY_EVENT_CREATED, + TELEMETRY_ITR_SKIPPED +} = require('../ci-visibility/telemetry') +const { CI_PROVIDER_NAME, GIT_REPOSITORY_URL, GIT_COMMIT_SHA, GIT_BRANCH } = require('./util/tags') +const { OS_VERSION, OS_PLATFORM, OS_ARCHITECTURE, RUNTIME_NAME, RUNTIME_VERSION } = require('./util/env') module.exports = class CiPlugin extends Plugin { constructor (...args) { @@ -71,6 +79,7 @@ module.exports = class CiPlugin extends Plugin { ...testSessionSpanMetadata } }) + this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'session') this.testModuleSpan = this.tracer.startSpan(`${this.constructor.id}.test_module`, { childOf: this.testSessionSpan, tags: { @@ -79,6 +88,7 @@ module.exports = class CiPlugin extends Plugin { ...testModuleSpanMetadata } }) + this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'module') }) this.addSub(`ci:${this.constructor.id}:itr:skipped-suites`, ({ skippedSuites, frameworkVersion }) => { @@ -97,25 +107,49 @@ module.exports = class CiPlugin extends Plugin { } }).finish() }) + this.telemetry.count(TELEMETRY_ITR_SKIPPED, { testLevel: 'suite' }, skippedSuites.length) }) } + get telemetry () { + const testFramework = this.constructor.id + return { + ciVisEvent: function (name, testLevel, tags = {}) { + incrementCountMetric(name, { + testLevel, + testFramework, + isUnsupportedCIProvider: this.isUnsupportedCIProvider, + ...tags + }) + }, + count: function (name, tags, value = 1) { + incrementCountMetric(name, tags, value) + }, + distribution: function (name, tags, measure) { + distributionMetric(name, tags, measure) + } + } + } + configure (config) { super.configure(config) this.testEnvironmentMetadata = getTestEnvironmentMetadata(this.constructor.id, this.config) this.codeOwnersEntries = getCodeOwnersFileEntries() const { - 'git.repository_url': repositoryUrl, - 'git.commit.sha': sha, - 'os.version': osVersion, - 'os.platform': osPlatform, - 'os.architecture': osArchitecture, - 'runtime.name': runtimeName, - 'runtime.version': runtimeVersion, - 'git.branch': branch + [GIT_REPOSITORY_URL]: repositoryUrl, + [GIT_COMMIT_SHA]: sha, + [OS_VERSION]: osVersion, + [OS_PLATFORM]: osPlatform, + [OS_ARCHITECTURE]: osArchitecture, + [RUNTIME_NAME]: runtimeName, + [RUNTIME_VERSION]: runtimeVersion, + [GIT_BRANCH]: branch, + [CI_PROVIDER_NAME]: ciProviderName } = this.testEnvironmentMetadata + this.isUnsupportedCIProvider = !ciProviderName + this.testConfiguration = { repositoryUrl, sha, @@ -170,6 +204,8 @@ module.exports = class CiPlugin extends Plugin { } } + this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'test', { hasCodeOwners: !!codeOwners }) + const testSpan = this.tracer .startSpan(`${this.constructor.id}.test`, { childOf, diff --git a/packages/dd-trace/src/plugins/util/exec.js b/packages/dd-trace/src/plugins/util/exec.js index a2d091232c6..3e3ca3f3660 100644 --- a/packages/dd-trace/src/plugins/util/exec.js +++ b/packages/dd-trace/src/plugins/util/exec.js @@ -1,10 +1,31 @@ const cp = require('child_process') const log = require('../../log') +const { distributionMetric, incrementCountMetric } = require('../../ci-visibility/telemetry') -const sanitizedExec = (cmd, flags, options = { stdio: 'pipe' }) => { +const sanitizedExec = ( + cmd, + flags, + operationMetric, + durationMetric, + errorMetric +) => { + let startTime + if (operationMetric) { + incrementCountMetric(operationMetric.name, operationMetric.tags) + } + if (durationMetric) { + startTime = Date.now() + } try { - return cp.execFileSync(cmd, flags, options).toString().replace(/(\r\n|\n|\r)/gm, '') + const result = cp.execFileSync(cmd, flags, { stdio: 'pipe' }).toString().replace(/(\r\n|\n|\r)/gm, '') + if (durationMetric) { + distributionMetric(durationMetric.name, durationMetric.tags, Date.now() - startTime) + } + return result } catch (e) { + if (errorMetric) { + incrementCountMetric(errorMetric.name, { ...errorMetric.tags, exitCode: e.status }) + } log.error(e) return '' } diff --git a/packages/dd-trace/src/plugins/util/git.js b/packages/dd-trace/src/plugins/util/git.js index 72ca5db3b59..885cbe5fb3c 100644 --- a/packages/dd-trace/src/plugins/util/git.js +++ b/packages/dd-trace/src/plugins/util/git.js @@ -1,10 +1,9 @@ -const { execFileSync } = require('child_process') +const cp = require('child_process') const os = require('os') const path = require('path') const fs = require('fs') const log = require('../../log') -const { sanitizedExec } = require('./exec') const { GIT_COMMIT_SHA, GIT_BRANCH, @@ -19,10 +18,46 @@ const { GIT_COMMIT_AUTHOR_NAME, CI_WORKSPACE_PATH } = require('./tags') +const { + incrementCountMetric, + distributionMetric, + TELEMETRY_GIT_COMMAND, + TELEMETRY_GIT_COMMAND_MS, + TELEMETRY_GIT_COMMAND_ERRORS +} = require('../../ci-visibility/telemetry') const { filterSensitiveInfoFromRepository } = require('./url') const GIT_REV_LIST_MAX_BUFFER = 8 * 1024 * 1024 // 8MB +function sanitizedExec ( + cmd, + flags, + operationMetric, + durationMetric, + errorMetric +) { + let startTime + if (operationMetric) { + incrementCountMetric(operationMetric.name, operationMetric.tags) + } + if (durationMetric) { + startTime = Date.now() + } + try { + const result = cp.execFileSync(cmd, flags, { stdio: 'pipe' }).toString().replace(/(\r\n|\n|\r)/gm, '') + if (durationMetric) { + distributionMetric(durationMetric.name, durationMetric.tags, Date.now() - startTime) + } + return result + } catch (e) { + if (errorMetric) { + incrementCountMetric(errorMetric.name, { ...errorMetric.tags, exitCode: e.status }) + } + log.error(e) + return '' + } +} + function isDirectory (path) { try { const stats = fs.statSync(path) @@ -33,7 +68,13 @@ function isDirectory (path) { } function isShallowRepository () { - return sanitizedExec('git', ['rev-parse', '--is-shallow-repository']) === 'true' + return sanitizedExec( + 'git', + ['rev-parse', '--is-shallow-repository'], + { name: TELEMETRY_GIT_COMMAND, tags: { command: 'check_shallow' } }, + { name: TELEMETRY_GIT_COMMAND_MS, tags: { command: 'check_shallow' } }, + { name: TELEMETRY_GIT_COMMAND_ERRORS, tags: { command: 'check_shallow' } } + ) === 'true' } function getGitVersion () { @@ -72,50 +113,76 @@ function unshallowRepository () { defaultRemoteName ] + incrementCountMetric(TELEMETRY_GIT_COMMAND, { command: 'unshallow' }) + const start = Date.now() try { - execFileSync('git', [ + cp.execFileSync('git', [ ...baseGitOptions, revParseHead ], { stdio: 'pipe' }) - } catch (e) { + } catch (err) { // If the local HEAD is a commit that has not been pushed to the remote, the above command will fail. - log.error(e) + log.error(err) + incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'unshallow', exitCode: err.status }) const upstreamRemote = sanitizedExec('git', ['rev-parse', '--abbrev-ref', '--symbolic-full-name', '@{upstream}']) try { - execFileSync('git', [ + cp.execFileSync('git', [ ...baseGitOptions, upstreamRemote ], { stdio: 'pipe' }) - } catch (e) { + } catch (err) { // If the CI is working on a detached HEAD or branch tracking hasn’t been set up, the above command will fail. - log.error(e) + log.error(err) + incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'unshallow', exitCode: err.status }) // We use sanitizedExec here because if this last option fails, we'll give up. - sanitizedExec('git', baseGitOptions) + sanitizedExec( + 'git', + baseGitOptions, + null, + null, + { name: TELEMETRY_GIT_COMMAND_ERRORS, tags: { command: 'unshallow' } } // we log the error in sanitizedExec + ) } } + distributionMetric(TELEMETRY_GIT_COMMAND_MS, { command: 'unshallow' }, Date.now() - start) } function getRepositoryUrl () { - return sanitizedExec('git', ['config', '--get', 'remote.origin.url']) + return sanitizedExec( + 'git', + ['config', '--get', 'remote.origin.url'], + { name: TELEMETRY_GIT_COMMAND, tags: { command: 'get_repository' } }, + { name: TELEMETRY_GIT_COMMAND_MS, tags: { command: 'get_repository' } }, + { name: TELEMETRY_GIT_COMMAND_ERRORS, tags: { command: 'get_repository' } } + ) } function getLatestCommits () { + incrementCountMetric(TELEMETRY_GIT_COMMAND, { command: 'get_local_commits' }) + const startTime = Date.now() try { - return execFileSync('git', ['log', '--format=%H', '-n 1000', '--since="1 month ago"'], { stdio: 'pipe' }) + const result = cp.execFileSync('git', ['log', '--format=%H', '-n 1000', '--since="1 month ago"'], { stdio: 'pipe' }) .toString() .split('\n') .filter(commit => commit) + distributionMetric(TELEMETRY_GIT_COMMAND_MS, { command: 'get_local_commits' }, Date.now() - startTime) + return result } catch (err) { log.error(`Get latest commits failed: ${err.message}`) + incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'get_local_commits', errorType: err.status }) return [] } } function getCommitsRevList (commitsToExclude, commitsToInclude) { + let result = [] + const commitsToExcludeString = commitsToExclude.map(commit => `^${commit}`) + incrementCountMetric(TELEMETRY_GIT_COMMAND, { command: 'get_objects' }) + const startTime = Date.now() try { - return execFileSync( + result = cp.execFileSync( 'git', [ 'rev-list', @@ -132,11 +199,14 @@ function getCommitsRevList (commitsToExclude, commitsToInclude) { .filter(commit => commit) } catch (err) { log.error(`Get commits to upload failed: ${err.message}`) - return [] + incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'get_objects', errorType: err.status }) } + distributionMetric(TELEMETRY_GIT_COMMAND_MS, { command: 'get_objects' }, Date.now() - startTime) + return result } function generatePackFilesForCommits (commitsToUpload) { + let result = [] const tmpFolder = os.tmpdir() if (!isDirectory(tmpFolder)) { @@ -148,10 +218,12 @@ function generatePackFilesForCommits (commitsToUpload) { const temporaryPath = path.join(tmpFolder, randomPrefix) const cwdPath = path.join(process.cwd(), randomPrefix) + incrementCountMetric(TELEMETRY_GIT_COMMAND, { command: 'pack_objects' }) + const startTime = Date.now() // Generates pack files to upload and // returns the ordered list of packfiles' paths function execGitPackObjects (targetPath) { - return execFileSync( + return cp.execFileSync( 'git', [ 'pack-objects', @@ -164,9 +236,10 @@ function generatePackFilesForCommits (commitsToUpload) { } try { - return execGitPackObjects(temporaryPath) + result = execGitPackObjects(temporaryPath) } catch (err) { log.error(err) + incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'pack_objects', errorType: err.status }) /** * The generation of pack files in the temporary folder (from `os.tmpdir()`) * sometimes fails in certain CI setups with the error message @@ -180,13 +253,15 @@ function generatePackFilesForCommits (commitsToUpload) { * TODO: fix issue and remove workaround. */ try { - return execGitPackObjects(cwdPath) + result = execGitPackObjects(cwdPath) } catch (err) { log.error(err) + incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'pack_objects', errorType: err.status }) } - - return [] } + distributionMetric(TELEMETRY_GIT_COMMAND_MS, { command: 'pack_objects' }, Date.now() - startTime) + + return result } // If there is ciMetadata, it takes precedence. diff --git a/packages/dd-trace/src/telemetry/index.js b/packages/dd-trace/src/telemetry/index.js index a2db5e13971..f7a300309b6 100644 --- a/packages/dd-trace/src/telemetry/index.js +++ b/packages/dd-trace/src/telemetry/index.js @@ -129,6 +129,10 @@ function onBeforeExit () { process.removeListener('beforeExit', onBeforeExit) const { reqType, payload } = createPayload('app-closing') sendData(config, application, host, reqType, payload) + // we flush before shutting down. Only in CI Visibility + if (config.isCiVisibility) { + metricsManager.send(config, application, host) + } } function createAppObject (config) { diff --git a/packages/dd-trace/test/ci-visibility/exporters/agentless/coverage-writer.spec.js b/packages/dd-trace/test/ci-visibility/exporters/agentless/coverage-writer.spec.js index 7bfdd197866..62e10e9753e 100644 --- a/packages/dd-trace/test/ci-visibility/exporters/agentless/coverage-writer.spec.js +++ b/packages/dd-trace/test/ci-visibility/exporters/agentless/coverage-writer.spec.js @@ -23,7 +23,8 @@ describe('CI Visibility Coverage Writer', () => { count: sinon.stub().returns(0), makePayload: sinon.stub().returns({ getHeaders: () => ({}), - pipe: () => {} + pipe: () => {}, + size: () => 1 }) } @@ -80,7 +81,8 @@ describe('CI Visibility Coverage Writer', () => { encoder.count.returns(2) const payload = { getHeaders: () => ({}), - pipe: () => {} + pipe: () => {}, + size: () => 1 } encoder.makePayload.returns(payload) @@ -101,7 +103,8 @@ describe('CI Visibility Coverage Writer', () => { const payload = { getHeaders: () => ({}), - pipe: () => {} + pipe: () => {}, + size: () => 1 } encoder.count.returns(1) diff --git a/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js b/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js index f3d331be567..54c182dd7c1 100644 --- a/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js +++ b/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js @@ -3,6 +3,7 @@ require('../../../../dd-trace/test/setup/tap') const cp = require('child_process') +const fs = require('fs') const CiVisibilityExporter = require('../../../src/ci-visibility/exporters/ci-visibility-exporter') const nock = require('nock') @@ -13,6 +14,7 @@ describe('CI Visibility Exporter', () => { beforeEach(() => { // to make sure `isShallowRepository` in `git.js` returns false sinon.stub(cp, 'execFileSync').returns('false') + sinon.stub(fs, 'readFileSync').returns('') process.env.DD_API_KEY = '1' nock.cleanAll() }) diff --git a/packages/dd-trace/test/plugins/util/git.spec.js b/packages/dd-trace/test/plugins/util/git.spec.js index c04bd371dbb..7f275287678 100644 --- a/packages/dd-trace/test/plugins/util/git.spec.js +++ b/packages/dd-trace/test/plugins/util/git.spec.js @@ -9,7 +9,6 @@ const path = require('path') const { GIT_REV_LIST_MAX_BUFFER } = require('../../../src/plugins/util/git') const proxyquire = require('proxyquire') -const sanitizedExecStub = sinon.stub().returns('') const execFileSyncStub = sinon.stub().returns('') const { @@ -29,9 +28,6 @@ const { const { getGitMetadata, unshallowRepository } = proxyquire('../../../src/plugins/util/git', { - './exec': { - sanitizedExec: sanitizedExecStub - }, 'child_process': { execFileSync: execFileSyncStub } @@ -47,7 +43,7 @@ function getFakeDirectory () { describe('git', () => { afterEach(() => { - sanitizedExecStub.reset() + execFileSyncStub.reset() delete process.env.DD_GIT_COMMIT_SHA delete process.env.DD_GIT_REPOSITORY_URL delete process.env.DD_GIT_BRANCH @@ -80,15 +76,15 @@ describe('git', () => { } ) expect(metadata[GIT_REPOSITORY_URL]).not.to.equal('ciRepositoryUrl') - expect(sanitizedExecStub).to.have.been.calledWith('git', ['ls-remote', '--get-url']) - expect(sanitizedExecStub).to.have.been.calledWith('git', ['show', '-s', '--format=%an,%ae,%aI,%cn,%ce,%cI']) - expect(sanitizedExecStub).not.to.have.been.calledWith('git', ['show', '-s', '--format=%s']) - expect(sanitizedExecStub).not.to.have.been.calledWith('git', ['rev-parse', 'HEAD']) - expect(sanitizedExecStub).not.to.have.been.calledWith('git', ['rev-parse', '--abbrev-ref', 'HEAD']) - expect(sanitizedExecStub).not.to.have.been.calledWith('git', ['rev-parse', '--show-toplevel']) + expect(execFileSyncStub).to.have.been.calledWith('git', ['ls-remote', '--get-url']) + expect(execFileSyncStub).to.have.been.calledWith('git', ['show', '-s', '--format=%an,%ae,%aI,%cn,%ce,%cI']) + expect(execFileSyncStub).not.to.have.been.calledWith('git', ['show', '-s', '--format=%s']) + expect(execFileSyncStub).not.to.have.been.calledWith('git', ['rev-parse', 'HEAD']) + expect(execFileSyncStub).not.to.have.been.calledWith('git', ['rev-parse', '--abbrev-ref', 'HEAD']) + expect(execFileSyncStub).not.to.have.been.calledWith('git', ['rev-parse', '--show-toplevel']) }) it('does not crash if git is not available', () => { - sanitizedExecStub.returns('') + execFileSyncStub.returns('') const ciMetadata = { repositoryUrl: 'https://github.com/datadog/safe-repository.git' } const metadata = getGitMetadata(ciMetadata) expect(metadata).to.eql({ @@ -107,7 +103,7 @@ describe('git', () => { }) }) it('returns all git metadata is git is available', () => { - sanitizedExecStub + execFileSyncStub .onCall(0).returns( 'git author,git.author@email.com,2022-02-14T16:22:03-05:00,' + 'git committer,git.committer@email.com,2022-02-14T16:23:03-05:00' @@ -133,12 +129,12 @@ describe('git', () => { [GIT_COMMIT_COMMITTER_NAME]: 'git committer', [CI_WORKSPACE_PATH]: 'ciWorkspacePath' }) - expect(sanitizedExecStub).to.have.been.calledWith('git', ['ls-remote', '--get-url']) - expect(sanitizedExecStub).to.have.been.calledWith('git', ['show', '-s', '--format=%s']) - expect(sanitizedExecStub).to.have.been.calledWith('git', ['show', '-s', '--format=%an,%ae,%aI,%cn,%ce,%cI']) - expect(sanitizedExecStub).to.have.been.calledWith('git', ['rev-parse', 'HEAD']) - expect(sanitizedExecStub).to.have.been.calledWith('git', ['rev-parse', '--abbrev-ref', 'HEAD']) - expect(sanitizedExecStub).to.have.been.calledWith('git', ['rev-parse', '--show-toplevel']) + expect(execFileSyncStub).to.have.been.calledWith('git', ['ls-remote', '--get-url']) + expect(execFileSyncStub).to.have.been.calledWith('git', ['show', '-s', '--format=%s']) + expect(execFileSyncStub).to.have.been.calledWith('git', ['show', '-s', '--format=%an,%ae,%aI,%cn,%ce,%cI']) + expect(execFileSyncStub).to.have.been.calledWith('git', ['rev-parse', 'HEAD']) + expect(execFileSyncStub).to.have.been.calledWith('git', ['rev-parse', '--abbrev-ref', 'HEAD']) + expect(execFileSyncStub).to.have.been.calledWith('git', ['rev-parse', '--show-toplevel']) }) }) @@ -246,11 +242,10 @@ describe('generatePackFilesForCommits', () => { describe('unshallowRepository', () => { afterEach(() => { - sanitizedExecStub.reset() execFileSyncStub.reset() }) it('works for the usual case', () => { - sanitizedExecStub + execFileSyncStub .onCall(0).returns( 'git version 2.39.0' ) @@ -271,16 +266,14 @@ describe('unshallowRepository', () => { expect(execFileSyncStub).to.have.been.calledWith('git', options) }) it('works if the local HEAD is a commit that has not been pushed to the remote', () => { - sanitizedExecStub + execFileSyncStub .onCall(0).returns( 'git version 2.39.0' ) .onCall(1).returns('origin') .onCall(2).returns('daede5785233abb1a3cb76b9453d4eb5b98290b3') - .onCall(3).returns('origin/master') - - execFileSyncStub - .onCall(0).throws() + .onCall(3).throws() + .onCall(4).returns('origin/master') const options = [ 'fetch', @@ -296,17 +289,15 @@ describe('unshallowRepository', () => { expect(execFileSyncStub).to.have.been.calledWith('git', options) }) it('works if the CI is working on a detached HEAD or branch tracking hasn’t been set up', () => { - sanitizedExecStub + execFileSyncStub .onCall(0).returns( 'git version 2.39.0' ) .onCall(1).returns('origin') .onCall(2).returns('daede5785233abb1a3cb76b9453d4eb5b98290b3') - .onCall(3).returns('origin/master') - - execFileSyncStub - .onCall(0).throws() - .onCall(1).throws() + .onCall(3).throws() + .onCall(4).returns('origin/master') + .onCall(5).throws() const options = [ 'fetch', @@ -318,17 +309,17 @@ describe('unshallowRepository', () => { ] unshallowRepository() - expect(sanitizedExecStub).to.have.been.calledWith('git', options) + expect(execFileSyncStub).to.have.been.calledWith('git', options) }) }) describe('user credentials', () => { afterEach(() => { - sanitizedExecStub.reset() + execFileSyncStub.reset() execFileSyncStub.reset() }) it('scrubs https user credentials', () => { - sanitizedExecStub + execFileSyncStub .onCall(0).returns( 'git author,git.author@email.com,2022-02-14T16:22:03-05:00,' + 'git committer,git.committer@email.com,2022-02-14T16:23:03-05:00' @@ -340,7 +331,7 @@ describe('user credentials', () => { .to.equal('https://github.com/datadog/safe-repository.git') }) it('scrubs ssh user credentials', () => { - sanitizedExecStub + execFileSyncStub .onCall(0).returns( 'git author,git.author@email.com,2022-02-14T16:22:03-05:00,' + 'git committer,git.committer@email.com,2022-02-14T16:23:03-05:00' From 2cd6c77b03282f197a4cb8494ab4e2dcafcac93d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Fri, 22 Dec 2023 15:55:17 +0100 Subject: [PATCH 04/21] [ci-visibility] Fix cucumber plugin tests for node<16 (#3902) --- packages/datadog-plugin-cucumber/test/index.spec.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/datadog-plugin-cucumber/test/index.spec.js b/packages/datadog-plugin-cucumber/test/index.spec.js index 71023f58d0f..33e7c753baa 100644 --- a/packages/datadog-plugin-cucumber/test/index.spec.js +++ b/packages/datadog-plugin-cucumber/test/index.spec.js @@ -1,6 +1,7 @@ 'use strict' const path = require('path') const { PassThrough } = require('stream') +const semver = require('semver') const proxyquire = require('proxyquire').noPreserveCache() const nock = require('nock') @@ -23,6 +24,7 @@ const { TEST_SOURCE_START } = require('../../dd-trace/src/plugins/util/test') +const { NODE_MAJOR } = require('../../../version') const { version: ddTraceVersion } = require('../../../package.json') const runCucumber = (version, Cucumber, requireName, featureName, testName) => { @@ -54,6 +56,9 @@ describe('Plugin', function () { let Cucumber this.timeout(10000) withVersions('cucumber', '@cucumber/cucumber', version => { + const specificVersion = require(`../../../versions/@cucumber/cucumber@${version}`).version() + if ((NODE_MAJOR <= 16) && semver.satisfies(specificVersion, '>=10')) return + afterEach(() => { // > If you want to run tests multiple times, you may need to clear Node's require cache // before subsequent calls in whichever manner best suits your needs. From 9dbac7a05204f02137f0ca15176935935ba8c89c Mon Sep 17 00:00:00 2001 From: Ayan Khan Date: Wed, 27 Dec 2023 15:00:57 -0500 Subject: [PATCH 05/21] add support for node specifiers (#3893) * add support for node specifiers --- .../src/child-process.js | 9 +- .../datadog-instrumentations/src/crypto.js | 3 +- packages/datadog-instrumentations/src/dns.js | 3 +- .../src/helpers/hooks.js | 9 +- .../src/helpers/instrument.js | 11 +- .../src/helpers/register.js | 20 +- .../src/http/client.js | 4 +- .../src/http/server.js | 11 +- .../src/http2/client.js | 4 +- .../src/http2/server.js | 4 +- packages/datadog-instrumentations/src/net.js | 12 +- .../datadog-plugin-dns/test/index.spec.js | 395 +++++++-------- .../test/integration-test/client.spec.js | 5 +- .../datadog-plugin-http/test/client.spec.js | 43 +- .../datadog-plugin-http/test/server.spec.js | 390 +++++++-------- .../datadog-plugin-http2/test/client.spec.js | 26 +- .../datadog-plugin-http2/test/server.spec.js | 320 ++++++------ .../datadog-plugin-net/test/index.spec.js | 454 +++++++++--------- packages/dd-trace/src/plugins/index.js | 5 + 19 files changed, 905 insertions(+), 823 deletions(-) diff --git a/packages/datadog-instrumentations/src/child-process.js b/packages/datadog-instrumentations/src/child-process.js index ba26dfdf7cf..3dca938ed42 100644 --- a/packages/datadog-instrumentations/src/child-process.js +++ b/packages/datadog-instrumentations/src/child-process.js @@ -9,11 +9,10 @@ const shimmer = require('../../datadog-shimmer') const childProcessChannel = channel('datadog:child_process:execution:start') const execMethods = ['exec', 'execFile', 'fork', 'spawn', 'execFileSync', 'execSync', 'spawnSync'] const names = ['child_process', 'node:child_process'] -names.forEach(name => { - addHook({ name }, childProcess => { - shimmer.massWrap(childProcess, execMethods, wrapChildProcessMethod()) - return childProcess - }) + +addHook({ name: names }, childProcess => { + shimmer.massWrap(childProcess, execMethods, wrapChildProcessMethod()) + return childProcess }) function wrapChildProcessMethod () { diff --git a/packages/datadog-instrumentations/src/crypto.js b/packages/datadog-instrumentations/src/crypto.js index 3113c16ef1d..7c95614cee7 100644 --- a/packages/datadog-instrumentations/src/crypto.js +++ b/packages/datadog-instrumentations/src/crypto.js @@ -11,8 +11,9 @@ const cryptoCipherCh = channel('datadog:crypto:cipher:start') const hashMethods = ['createHash', 'createHmac', 'createSign', 'createVerify', 'sign', 'verify'] const cipherMethods = ['createCipheriv', 'createDecipheriv'] +const names = ['crypto', 'node:crypto'] -addHook({ name: 'crypto' }, crypto => { +addHook({ name: names }, crypto => { shimmer.massWrap(crypto, hashMethods, wrapCryptoMethod(cryptoHashCh)) shimmer.massWrap(crypto, cipherMethods, wrapCryptoMethod(cryptoCipherCh)) return crypto diff --git a/packages/datadog-instrumentations/src/dns.js b/packages/datadog-instrumentations/src/dns.js index 7c4f18c22b7..de827ea5182 100644 --- a/packages/datadog-instrumentations/src/dns.js +++ b/packages/datadog-instrumentations/src/dns.js @@ -18,8 +18,9 @@ const rrtypes = { } const rrtypeMap = new WeakMap() +const names = ['dns', 'node:dns'] -addHook({ name: 'dns' }, dns => { +addHook({ name: names }, dns => { dns.lookup = wrap('apm:dns:lookup', dns.lookup, 2) dns.lookupService = wrap('apm:dns:lookup_service', dns.lookupService, 3) dns.resolve = wrap('apm:dns:resolve', dns.resolve, 2) diff --git a/packages/datadog-instrumentations/src/helpers/hooks.js b/packages/datadog-instrumentations/src/helpers/hooks.js index ad572e41090..702084b23a9 100644 --- a/packages/datadog-instrumentations/src/helpers/hooks.js +++ b/packages/datadog-instrumentations/src/helpers/hooks.js @@ -31,7 +31,6 @@ module.exports = { 'bunyan': () => require('../bunyan'), 'cassandra-driver': () => require('../cassandra-driver'), 'child_process': () => require('../child-process'), - 'node:child_process': () => require('../child-process'), 'connect': () => require('../connect'), 'cookie': () => require('../cookie'), 'cookie-parser': () => require('../cookie-parser'), @@ -45,7 +44,6 @@ module.exports = { 'fastify': () => require('../fastify'), 'find-my-way': () => require('../find-my-way'), 'fs': () => require('../fs'), - 'node:fs': () => require('../fs'), 'generic-pool': () => require('../generic-pool'), 'graphql': () => require('../graphql'), 'grpc': () => require('../grpc'), @@ -79,6 +77,13 @@ module.exports = { 'mysql2': () => require('../mysql2'), 'net': () => require('../net'), 'next': () => require('../next'), + 'node:child_process': () => require('../child-process'), + 'node:crypto': () => require('../crypto'), + 'node:dns': () => require('../dns'), + 'node:http': () => require('../http'), + 'node:http2': () => require('../http2'), + 'node:https': () => require('../http'), + 'node:net': () => require('../net'), 'oracledb': () => require('../oracledb'), 'openai': () => require('../openai'), 'paperplane': () => require('../paperplane'), diff --git a/packages/datadog-instrumentations/src/helpers/instrument.js b/packages/datadog-instrumentations/src/helpers/instrument.js index 323c6b01624..0ca8b63df48 100644 --- a/packages/datadog-instrumentations/src/helpers/instrument.js +++ b/packages/datadog-instrumentations/src/helpers/instrument.js @@ -21,11 +21,16 @@ exports.channel = function (name) { * @param Function hook */ exports.addHook = function addHook ({ name, versions, file }, hook) { - if (!instrumentations[name]) { - instrumentations[name] = [] + if (typeof name === 'string') { + name = [name] } - instrumentations[name].push({ name, versions, file, hook }) + for (const val of name) { + if (!instrumentations[val]) { + instrumentations[val] = [] + } + instrumentations[val].push({ name: val, versions, file, hook }) + } } // AsyncResource.bind exists and binds `this` properly only from 17.8.0 and up. diff --git a/packages/datadog-instrumentations/src/helpers/register.js b/packages/datadog-instrumentations/src/helpers/register.js index e89a91b55f2..9fc22ca45f1 100644 --- a/packages/datadog-instrumentations/src/helpers/register.js +++ b/packages/datadog-instrumentations/src/helpers/register.js @@ -24,6 +24,7 @@ if (!disabledInstrumentations.has('fetch')) { require('../fetch') } +const HOOK_SYMBOL = Symbol('hookExportsMap') // TODO: make this more efficient for (const packageName of names) { @@ -42,14 +43,29 @@ for (const packageName of names) { for (const { name, file, versions, hook } of instrumentations[packageName]) { const fullFilename = filename(name, file) + // Create a WeakMap associated with the hook function so that patches on the same moduleExport only happens once + // for example by instrumenting both dns and node:dns double the spans would be created + // since they both patch the same moduleExport, this WeakMap is used to mitigate that + if (!hook[HOOK_SYMBOL]) { + hook[HOOK_SYMBOL] = new WeakMap() + } + if (moduleName === fullFilename) { const version = moduleVersion || getVersion(moduleBaseDir) if (matchVersion(version, versions)) { + // Check if the hook already has a set moduleExport + if (hook[HOOK_SYMBOL].has(moduleExports)) { + return moduleExports + } + try { loadChannel.publish({ name, version, file }) - - moduleExports = hook(moduleExports, version) + // Send the name and version of the module back to the callback because now addHook + // takes in an array of names so by passing the name the callback will know which module name is being used + moduleExports = hook(moduleExports, version, name) + // Set the moduleExports in the hooks weakmap + hook[HOOK_SYMBOL].set(moduleExports, name) } catch (e) { log.error(e) } diff --git a/packages/datadog-instrumentations/src/http/client.js b/packages/datadog-instrumentations/src/http/client.js index fcf5cc05f0a..89e621e642d 100644 --- a/packages/datadog-instrumentations/src/http/client.js +++ b/packages/datadog-instrumentations/src/http/client.js @@ -14,9 +14,9 @@ const endChannel = channel('apm:http:client:request:end') const asyncStartChannel = channel('apm:http:client:request:asyncStart') const errorChannel = channel('apm:http:client:request:error') -addHook({ name: 'https' }, hookFn) +const names = ['http', 'https', 'node:http', 'node:https'] -addHook({ name: 'http' }, hookFn) +addHook({ name: names }, hookFn) function hookFn (http) { patch(http, 'request') diff --git a/packages/datadog-instrumentations/src/http/server.js b/packages/datadog-instrumentations/src/http/server.js index f3eb528214f..680e6b8dcbf 100644 --- a/packages/datadog-instrumentations/src/http/server.js +++ b/packages/datadog-instrumentations/src/http/server.js @@ -15,14 +15,17 @@ const finishSetHeaderCh = channel('datadog:http:server:response:set-header:finis const requestFinishedSet = new WeakSet() -addHook({ name: 'https' }, http => { - // http.ServerResponse not present on https +const httpNames = ['http', 'node:http'] +const httpsNames = ['https', 'node:https'] + +addHook({ name: httpNames }, http => { + shimmer.wrap(http.ServerResponse.prototype, 'emit', wrapResponseEmit) shimmer.wrap(http.Server.prototype, 'emit', wrapEmit) return http }) -addHook({ name: 'http' }, http => { - shimmer.wrap(http.ServerResponse.prototype, 'emit', wrapResponseEmit) +addHook({ name: httpsNames }, http => { + // http.ServerResponse not present on https shimmer.wrap(http.Server.prototype, 'emit', wrapEmit) return http }) diff --git a/packages/datadog-instrumentations/src/http2/client.js b/packages/datadog-instrumentations/src/http2/client.js index de4957318ae..651c9ed6edd 100644 --- a/packages/datadog-instrumentations/src/http2/client.js +++ b/packages/datadog-instrumentations/src/http2/client.js @@ -10,6 +10,8 @@ const asyncStartChannel = channel('apm:http2:client:request:asyncStart') const asyncEndChannel = channel('apm:http2:client:request:asyncEnd') const errorChannel = channel('apm:http2:client:request:error') +const names = ['http2', 'node:http2'] + function createWrapEmit (ctx) { return function wrapEmit (emit) { return function (event, arg1) { @@ -66,7 +68,7 @@ function wrapConnect (connect) { } } -addHook({ name: 'http2' }, http2 => { +addHook({ name: names }, http2 => { shimmer.wrap(http2, 'connect', wrapConnect) return http2 diff --git a/packages/datadog-instrumentations/src/http2/server.js b/packages/datadog-instrumentations/src/http2/server.js index 6c9a290c7a7..07bfa11e453 100644 --- a/packages/datadog-instrumentations/src/http2/server.js +++ b/packages/datadog-instrumentations/src/http2/server.js @@ -14,7 +14,9 @@ const startServerCh = channel('apm:http2:server:request:start') const errorServerCh = channel('apm:http2:server:request:error') const finishServerCh = channel('apm:http2:server:request:finish') -addHook({ name: 'http2' }, http2 => { +const names = ['http2', 'node:http2'] + +addHook({ name: names }, http2 => { shimmer.wrap(http2, 'createSecureServer', wrapCreateServer) shimmer.wrap(http2, 'createServer', wrapCreateServer) return http2 diff --git a/packages/datadog-instrumentations/src/net.js b/packages/datadog-instrumentations/src/net.js index a5de6f511ba..e2e6ecaefe7 100644 --- a/packages/datadog-instrumentations/src/net.js +++ b/packages/datadog-instrumentations/src/net.js @@ -17,8 +17,16 @@ const errorTCPCh = channel('apm:net:tcp:error') const connectionCh = channel(`apm:net:tcp:connection`) -addHook({ name: 'net' }, net => { - require('dns') +const names = ['net', 'node:net'] + +addHook({ name: names }, (net, version, name) => { + // explicitly require dns so that net gets an instrumented instance + // so that we don't miss the dns calls + if (name === 'net') { + require('dns') + } else { + require('node:dns') + } shimmer.wrap(net.Socket.prototype, 'connect', connect => function () { if (!startICPCh.hasSubscribers || !startTCPCh.hasSubscribers) { diff --git a/packages/datadog-plugin-dns/test/index.spec.js b/packages/datadog-plugin-dns/test/index.spec.js index 5b2ab06ecec..3550e06a059 100644 --- a/packages/datadog-plugin-dns/test/index.spec.js +++ b/packages/datadog-plugin-dns/test/index.spec.js @@ -5,233 +5,236 @@ const { promisify } = require('util') const { storage } = require('../../datadog-core') const { ERROR_TYPE, ERROR_MESSAGE } = require('../../dd-trace/src/constants') +const PLUGINS = ['dns', 'node:dns'] + describe('Plugin', () => { let dns let tracer + PLUGINS.forEach(plugin => { + describe(plugin, () => { + afterEach(() => { + return agent.close() + }) - describe('dns', () => { - afterEach(() => { - return agent.close() - }) - - beforeEach(() => { - return agent.load('dns') - .then(() => { - dns = require('dns') - tracer = require('../../dd-trace') - }) - }) - - it('should instrument lookup', done => { - agent - .use(traces => { - expect(traces[0][0]).to.deep.include({ - name: 'dns.lookup', - service: 'test', - resource: 'localhost' + beforeEach(() => { + return agent.load('dns') + .then(() => { + dns = require(plugin) + tracer = require('../../dd-trace') }) - expect(traces[0][0].meta).to.deep.include({ - 'component': 'dns', - 'span.kind': 'client', - 'dns.hostname': 'localhost', - 'dns.address': '127.0.0.1' - }) - }) - .then(done) - .catch(done) - - dns.lookup('localhost', 4, (err, address, family) => err && done(err)) - }) + }) - it('should instrument lookup with all addresses', done => { - agent - .use(traces => { - expect(traces[0][0]).to.deep.include({ - name: 'dns.lookup', - service: 'test', - resource: 'localhost' - }) - expect(traces[0][0].meta).to.deep.include({ - 'component': 'dns', - 'span.kind': 'client', - 'dns.hostname': 'localhost', - 'dns.address': '127.0.0.1', - 'dns.addresses': '127.0.0.1,::1' - }) - }) - .then(done) - .catch(done) + it('should instrument lookup', done => { + agent + .use(traces => { + expect(traces[0][0]).to.deep.include({ + name: 'dns.lookup', + service: 'test', + resource: 'localhost' + }) + expect(traces[0][0].meta).to.deep.include({ + 'component': 'dns', + 'span.kind': 'client', + 'dns.hostname': 'localhost', + 'dns.address': '127.0.0.1' + }) + }) + .then(done) + .catch(done) + + dns.lookup('localhost', 4, (err, address, family) => err && done(err)) + }) - dns.lookup('localhost', { all: true }, (err, address, family) => err && done(err)) - }) + it('should instrument lookup with all addresses', done => { + agent + .use(traces => { + expect(traces[0][0]).to.deep.include({ + name: 'dns.lookup', + service: 'test', + resource: 'localhost' + }) + expect(traces[0][0].meta).to.deep.include({ + 'component': 'dns', + 'span.kind': 'client', + 'dns.hostname': 'localhost', + 'dns.address': '127.0.0.1', + 'dns.addresses': '127.0.0.1,::1' + }) + }) + .then(done) + .catch(done) + + dns.lookup('localhost', { all: true }, (err, address, family) => err && done(err)) + }) - it('should instrument errors correctly', done => { - agent - .use(traces => { - expect(traces[0][0]).to.deep.include({ - name: 'dns.lookup', - service: 'test', - resource: 'fakedomain.faketld', - error: 1 - }) - expect(traces[0][0].meta).to.deep.include({ - 'component': 'dns', - 'span.kind': 'client', - 'dns.hostname': 'fakedomain.faketld', - [ERROR_TYPE]: 'Error', - [ERROR_MESSAGE]: 'getaddrinfo ENOTFOUND fakedomain.faketld' - }) + it('should instrument errors correctly', done => { + agent + .use(traces => { + expect(traces[0][0]).to.deep.include({ + name: 'dns.lookup', + service: 'test', + resource: 'fakedomain.faketld', + error: 1 + }) + expect(traces[0][0].meta).to.deep.include({ + 'component': 'dns', + 'span.kind': 'client', + 'dns.hostname': 'fakedomain.faketld', + [ERROR_TYPE]: 'Error', + [ERROR_MESSAGE]: 'getaddrinfo ENOTFOUND fakedomain.faketld' + }) + }) + .then(done) + .catch(done) + + dns.lookup('fakedomain.faketld', 4, (err, address, family) => { + expect(err).to.not.be.null }) - .then(done) - .catch(done) + }) - dns.lookup('fakedomain.faketld', 4, (err, address, family) => { - expect(err).to.not.be.null + it('should instrument lookupService', done => { + agent + .use(traces => { + expect(traces[0][0]).to.deep.include({ + name: 'dns.lookup_service', + service: 'test', + resource: '127.0.0.1:22' + }) + expect(traces[0][0].meta).to.deep.include({ + 'component': 'dns', + 'span.kind': 'client', + 'dns.address': '127.0.0.1' + }) + expect(traces[0][0].metrics).to.deep.include({ + 'dns.port': 22 + }) + }) + .then(done) + .catch(done) + + dns.lookupService('127.0.0.1', 22, err => err && done(err)) }) - }) - it('should instrument lookupService', done => { - agent - .use(traces => { - expect(traces[0][0]).to.deep.include({ - name: 'dns.lookup_service', - service: 'test', - resource: '127.0.0.1:22' - }) - expect(traces[0][0].meta).to.deep.include({ - 'component': 'dns', - 'span.kind': 'client', - 'dns.address': '127.0.0.1' - }) - expect(traces[0][0].metrics).to.deep.include({ - 'dns.port': 22 - }) - }) - .then(done) - .catch(done) + it('should instrument resolve', done => { + agent + .use(traces => { + expect(traces[0][0]).to.deep.include({ + name: 'dns.resolve', + service: 'test', + resource: 'A lvh.me' + }) + expect(traces[0][0].meta).to.deep.include({ + 'component': 'dns', + 'span.kind': 'client', + 'dns.hostname': 'lvh.me', + 'dns.rrtype': 'A' + }) + }) + .then(done) + .catch(done) + + dns.resolve('lvh.me', err => err && done(err)) + }) - dns.lookupService('127.0.0.1', 22, err => err && done(err)) - }) + it('should instrument resolve shorthands', done => { + agent + .use(traces => { + expect(traces[0][0]).to.deep.include({ + name: 'dns.resolve', + service: 'test', + resource: 'ANY lvh.me' + }) + expect(traces[0][0].meta).to.deep.include({ + 'component': 'dns', + 'span.kind': 'client', + 'dns.hostname': 'lvh.me', + 'dns.rrtype': 'ANY' + }) + }) + .then(done) + .catch(done) + + dns.resolveAny('lvh.me', err => err && done(err)) + }) - it('should instrument resolve', done => { - agent - .use(traces => { - expect(traces[0][0]).to.deep.include({ - name: 'dns.resolve', - service: 'test', - resource: 'A lvh.me' - }) - expect(traces[0][0].meta).to.deep.include({ - 'component': 'dns', - 'span.kind': 'client', - 'dns.hostname': 'lvh.me', - 'dns.rrtype': 'A' - }) - }) - .then(done) - .catch(done) + it('should instrument reverse', done => { + agent + .use(traces => { + expect(traces[0][0]).to.deep.include({ + name: 'dns.reverse', + service: 'test', + resource: '127.0.0.1' + }) + expect(traces[0][0].meta).to.deep.include({ + 'component': 'dns', + 'span.kind': 'client', + 'dns.ip': '127.0.0.1' + }) + }) + .then(done) + .catch(done) + + dns.reverse('127.0.0.1', err => err && done(err)) + }) - dns.resolve('lvh.me', err => err && done(err)) - }) + it('should preserve the parent scope in the callback', done => { + const span = tracer.startSpan('dummySpan', {}) - it('should instrument resolve shorthands', done => { - agent - .use(traces => { - expect(traces[0][0]).to.deep.include({ - name: 'dns.resolve', - service: 'test', - resource: 'ANY lvh.me' - }) - expect(traces[0][0].meta).to.deep.include({ - 'component': 'dns', - 'span.kind': 'client', - 'dns.hostname': 'lvh.me', - 'dns.rrtype': 'ANY' - }) - }) - .then(done) - .catch(done) + tracer.scope().activate(span, () => { + dns.lookup('localhost', 4, (err) => { + if (err) return done(err) - dns.resolveAny('lvh.me', err => err && done(err)) - }) + expect(tracer.scope().active()).to.equal(span) - it('should instrument reverse', done => { - agent - .use(traces => { - expect(traces[0][0]).to.deep.include({ - name: 'dns.reverse', - service: 'test', - resource: '127.0.0.1' - }) - expect(traces[0][0].meta).to.deep.include({ - 'component': 'dns', - 'span.kind': 'client', - 'dns.ip': '127.0.0.1' + done() }) }) - .then(done) - .catch(done) - - dns.reverse('127.0.0.1', err => err && done(err)) - }) - - it('should preserve the parent scope in the callback', done => { - const span = tracer.startSpan('dummySpan', {}) - - tracer.scope().activate(span, () => { - dns.lookup('localhost', 4, (err) => { - if (err) return done(err) + }) - expect(tracer.scope().active()).to.equal(span) + it('should work with promisify', () => { + const lookup = promisify(dns.lookup) - done() + return lookup('localhost', 4).then(({ address, family }) => { + expect(address).to.equal('127.0.0.1') + expect(family).to.equal(4) }) }) - }) - - it('should work with promisify', () => { - const lookup = promisify(dns.lookup) - return lookup('localhost', 4).then(({ address, family }) => { - expect(address).to.equal('127.0.0.1') - expect(family).to.equal(4) + it('should instrument Resolver', done => { + const resolver = new dns.Resolver() + + agent + .use(traces => { + expect(traces[0][0]).to.deep.include({ + name: 'dns.resolve', + service: 'test', + resource: 'A lvh.me' + }) + expect(traces[0][0].meta).to.deep.include({ + 'component': 'dns', + 'dns.hostname': 'lvh.me', + 'dns.rrtype': 'A' + }) + }) + .then(done) + .catch(done) + + resolver.resolve('lvh.me', err => err && done(err)) }) - }) - it('should instrument Resolver', done => { - const resolver = new dns.Resolver() + it('should skip instrumentation for noop context', done => { + const resolver = new dns.Resolver() + const timer = setTimeout(done, 200) - agent - .use(traces => { - expect(traces[0][0]).to.deep.include({ - name: 'dns.resolve', - service: 'test', - resource: 'A lvh.me' + agent + .use(() => { + done(new Error('Resolve was traced.')) + clearTimeout(timer) }) - expect(traces[0][0].meta).to.deep.include({ - 'component': 'dns', - 'dns.hostname': 'lvh.me', - 'dns.rrtype': 'A' - }) - }) - .then(done) - .catch(done) - - resolver.resolve('lvh.me', err => err && done(err)) - }) - - it('should skip instrumentation for noop context', done => { - const resolver = new dns.Resolver() - const timer = setTimeout(done, 200) - agent - .use(() => { - done(new Error('Resolve was traced.')) - clearTimeout(timer) + storage.run({ noop: true }, () => { + resolver.resolve('lvh.me', () => {}) }) - - storage.run({ noop: true }, () => { - resolver.resolve('lvh.me', () => {}) }) }) }) diff --git a/packages/datadog-plugin-fastify/test/integration-test/client.spec.js b/packages/datadog-plugin-fastify/test/integration-test/client.spec.js index 4dce20e0255..581f512305b 100644 --- a/packages/datadog-plugin-fastify/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-fastify/test/integration-test/client.spec.js @@ -14,9 +14,8 @@ describe('esm', () => { let proc let sandbox - // TODO: fastify instrumentation breaks with esm for version 4.23.2 but works for commonJS, - // fix it and change the versions tested - withVersions('fastify', 'fastify', '^3', version => { + // skip older versions of fastify due to syntax differences + withVersions('fastify', 'fastify', '>=3', version => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'fastify@${version}'`], false, diff --git a/packages/datadog-plugin-http/test/client.spec.js b/packages/datadog-plugin-http/test/client.spec.js index 7256950ac83..5a48959892e 100644 --- a/packages/datadog-plugin-http/test/client.spec.js +++ b/packages/datadog-plugin-http/test/client.spec.js @@ -25,15 +25,21 @@ describe('Plugin', () => { let appListener let tracer - ['http', 'https'].forEach(protocol => { - describe(protocol, () => { + ['http', 'https', 'node:http', 'node:https'].forEach(pluginToBeLoaded => { + const protocol = pluginToBeLoaded.split(':')[1] || pluginToBeLoaded + describe(pluginToBeLoaded, () => { function server (app, port, listener) { let server - if (protocol === 'https') { + if (pluginToBeLoaded === 'https') { process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0' server = require('https').createServer({ key, cert }, app) - } else { + } else if (pluginToBeLoaded === 'node:https') { + process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0' + server = require('node:https').createServer({ key, cert }, app) + } else if (pluginToBeLoaded === 'http') { server = require('http').createServer(app) + } else { + server = require('node:http').createServer(app) } server.listen(port, 'localhost', listener) return server @@ -55,7 +61,7 @@ describe('Plugin', () => { beforeEach(() => { return agent.load('http', { server: false }) .then(() => { - http = require(protocol) + http = require(pluginToBeLoaded) express = require('express') }) }) @@ -904,8 +910,13 @@ describe('Plugin', () => { it('should only record a request once', done => { // Make sure both plugins are loaded, which could cause double-counting. - require('http') - require('https') + if (pluginToBeLoaded.includes('node:')) { + require('node:http') + require('node:https') + } else { + require('http') + require('https') + } const app = express() @@ -1072,7 +1083,7 @@ describe('Plugin', () => { ch = require('dc-polyfill').channel('apm:http:client:request:start') sub = () => {} tracer = require('../../dd-trace') - http = require(protocol) + http = require(pluginToBeLoaded) }) }) @@ -1119,7 +1130,7 @@ describe('Plugin', () => { return agent.load('http', config) .then(() => { - http = require(protocol) + http = require(pluginToBeLoaded) express = require('express') }) }) @@ -1160,7 +1171,7 @@ describe('Plugin', () => { return agent.load('http', config) .then(() => { - http = require(protocol) + http = require(pluginToBeLoaded) express = require('express') }) }) @@ -1209,7 +1220,7 @@ describe('Plugin', () => { return agent.load('http', config) .then(() => { - http = require(protocol) + http = require(pluginToBeLoaded) express = require('express') }) }) @@ -1254,7 +1265,7 @@ describe('Plugin', () => { return agent.load('http', config) .then(() => { - http = require(protocol) + http = require(pluginToBeLoaded) express = require('express') }) }) @@ -1326,7 +1337,7 @@ describe('Plugin', () => { return agent.load('http', config) .then(() => { - http = require(protocol) + http = require(pluginToBeLoaded) express = require('express') }) }) @@ -1439,7 +1450,7 @@ describe('Plugin', () => { return agent.load('http', config) .then(() => { - http = require(protocol) + http = require(pluginToBeLoaded) express = require('express') }) }) @@ -1485,7 +1496,7 @@ describe('Plugin', () => { return agent.load('http', config) .then(() => { - http = require(protocol) + http = require(pluginToBeLoaded) express = require('express') }) }) @@ -1532,7 +1543,7 @@ describe('Plugin', () => { return agent.load('http', config) .then(() => { - http = require(protocol) + http = require(pluginToBeLoaded) express = require('express') }) }) diff --git a/packages/datadog-plugin-http/test/server.spec.js b/packages/datadog-plugin-http/test/server.spec.js index 9a0135ea967..4f6f5a15ed3 100644 --- a/packages/datadog-plugin-http/test/server.spec.js +++ b/packages/datadog-plugin-http/test/server.spec.js @@ -15,248 +15,250 @@ describe('Plugin', () => { let port let app - describe('http/server', () => { - beforeEach(() => { - tracer = require('../../dd-trace') - listener = (req, res) => { - app && app(req, res) - res.writeHead(200) - res.end() - } - }) - - beforeEach(() => { - return getPort().then(newPort => { - port = newPort - }) - }) - - afterEach(() => { - appListener && appListener.close() - app = null - return agent.close({ ritmReset: false }) - }) - - describe('canceled request', () => { + ['http', 'node:http'].forEach(pluginToBeLoaded => { + describe(`${pluginToBeLoaded}/server`, () => { beforeEach(() => { + tracer = require('../../dd-trace') listener = (req, res) => { - setTimeout(() => { - app && app(req, res) - res.writeHead(200) - res.end() - }, 500) + app && app(req, res) + res.writeHead(200) + res.end() } }) beforeEach(() => { - return agent.load('http') - .then(() => { - http = require('http') - }) - }) - - beforeEach(done => { - const server = new http.Server(listener) - appListener = server - .listen(port, 'localhost', () => done()) + return getPort().then(newPort => { + port = newPort + }) }) - it('should send traces to agent', (done) => { - app = sinon.stub() - agent - .use(traces => { - expect(app).not.to.have.been.called // request should be cancelled before call to app - expect(traces[0][0]).to.have.property('name', 'web.request') - expect(traces[0][0]).to.have.property('service', 'test') - expect(traces[0][0]).to.have.property('type', 'web') - expect(traces[0][0]).to.have.property('resource', 'GET') - expect(traces[0][0].meta).to.have.property('span.kind', 'server') - expect(traces[0][0].meta).to.have.property('http.url', `http://localhost:${port}/user`) - expect(traces[0][0].meta).to.have.property('http.method', 'GET') - expect(traces[0][0].meta).to.have.property('http.status_code', '200') - expect(traces[0][0].meta).to.have.property('component', 'http') - }) - .then(done) - .catch(done) - const source = axios.CancelToken.source() - axios.get(`http://localhost:${port}/user`, { cancelToken: source.token }) - .then(() => {}) - setTimeout(() => { source.cancel() }, 100) + afterEach(() => { + appListener && appListener.close() + app = null + return agent.close({ ritmReset: false }) }) - }) - describe('without configuration', () => { - beforeEach(() => { - return agent.load('http') - .then(() => { - http = require('http') - }) - }) + describe('canceled request', () => { + beforeEach(() => { + listener = (req, res) => { + setTimeout(() => { + app && app(req, res) + res.writeHead(200) + res.end() + }, 500) + } + }) - beforeEach(done => { - const server = new http.Server(listener) - appListener = server - .listen(port, 'localhost', () => done()) - }) + beforeEach(() => { + return agent.load('http') + .then(() => { + http = require(pluginToBeLoaded) + }) + }) - withNamingSchema( - done => { - axios.get(`http://localhost:${port}/user`).catch(done) - }, - rawExpectedSchema.server - ) - - it('should do automatic instrumentation', done => { - agent - .use(traces => { - expect(traces[0][0]).to.have.property('name', 'web.request') - expect(traces[0][0]).to.have.property('service', 'test') - expect(traces[0][0]).to.have.property('type', 'web') - expect(traces[0][0]).to.have.property('resource', 'GET') - expect(traces[0][0].meta).to.have.property('span.kind', 'server') - expect(traces[0][0].meta).to.have.property('http.url', `http://localhost:${port}/user`) - expect(traces[0][0].meta).to.have.property('http.method', 'GET') - expect(traces[0][0].meta).to.have.property('http.status_code', '200') - expect(traces[0][0].meta).to.have.property('component', 'http') - }) - .then(done) - .catch(done) + beforeEach(done => { + const server = new http.Server(listener) + appListener = server + .listen(port, 'localhost', () => done()) + }) - axios.get(`http://localhost:${port}/user`).catch(done) + it('should send traces to agent', (done) => { + app = sinon.stub() + agent + .use(traces => { + expect(app).not.to.have.been.called // request should be cancelled before call to app + expect(traces[0][0]).to.have.property('name', 'web.request') + expect(traces[0][0]).to.have.property('service', 'test') + expect(traces[0][0]).to.have.property('type', 'web') + expect(traces[0][0]).to.have.property('resource', 'GET') + expect(traces[0][0].meta).to.have.property('span.kind', 'server') + expect(traces[0][0].meta).to.have.property('http.url', `http://localhost:${port}/user`) + expect(traces[0][0].meta).to.have.property('http.method', 'GET') + expect(traces[0][0].meta).to.have.property('http.status_code', '200') + expect(traces[0][0].meta).to.have.property('component', 'http') + }) + .then(done) + .catch(done) + const source = axios.CancelToken.source() + axios.get(`http://localhost:${port}/user`, { cancelToken: source.token }) + .then(() => {}) + setTimeout(() => { source.cancel() }, 100) + }) }) - it('should run the request listener in the request scope', done => { - const spy = sinon.spy(() => { - expect(tracer.scope().active()).to.not.be.null + describe('without configuration', () => { + beforeEach(() => { + return agent.load('http') + .then(() => { + http = require(pluginToBeLoaded) + }) }) - incomingHttpRequestStart.subscribe(spy) + beforeEach(done => { + const server = new http.Server(listener) + appListener = server + .listen(port, 'localhost', () => done()) + }) - app = (req, res) => { - expect(tracer.scope().active()).to.not.be.null + withNamingSchema( + done => { + axios.get(`http://localhost:${port}/user`).catch(done) + }, + rawExpectedSchema.server + ) + + it('should do automatic instrumentation', done => { + agent + .use(traces => { + expect(traces[0][0]).to.have.property('name', 'web.request') + expect(traces[0][0]).to.have.property('service', 'test') + expect(traces[0][0]).to.have.property('type', 'web') + expect(traces[0][0]).to.have.property('resource', 'GET') + expect(traces[0][0].meta).to.have.property('span.kind', 'server') + expect(traces[0][0].meta).to.have.property('http.url', `http://localhost:${port}/user`) + expect(traces[0][0].meta).to.have.property('http.method', 'GET') + expect(traces[0][0].meta).to.have.property('http.status_code', '200') + expect(traces[0][0].meta).to.have.property('component', 'http') + }) + .then(done) + .catch(done) - const abortController = new AbortController() - expect(spy).to.have.been.calledOnceWithExactly({ req, res, abortController }, incomingHttpRequestStart.name) + axios.get(`http://localhost:${port}/user`).catch(done) + }) - done() - } + it('should run the request listener in the request scope', done => { + const spy = sinon.spy(() => { + expect(tracer.scope().active()).to.not.be.null + }) - axios.get(`http://localhost:${port}/user`).catch(done) - }) + incomingHttpRequestStart.subscribe(spy) + + app = (req, res) => { + expect(tracer.scope().active()).to.not.be.null + + const abortController = new AbortController() + expect(spy).to.have.been.calledOnceWithExactly({ req, res, abortController }, incomingHttpRequestStart.name) - it(`should run the request's close event in the correct context`, done => { - app = (req, res) => { - req.on('close', () => { - expect(tracer.scope().active()).to.equal(null) done() - }) - } + } - axios.get(`http://localhost:${port}/user`).catch(done) - }) + axios.get(`http://localhost:${port}/user`).catch(done) + }) - it(`should run the response's close event in the correct context`, done => { - app = (req, res) => { - const span = tracer.scope().active() + it(`should run the request's close event in the correct context`, done => { + app = (req, res) => { + req.on('close', () => { + expect(tracer.scope().active()).to.equal(null) + done() + }) + } - res.on('close', () => { - expect(tracer.scope().active()).to.equal(span) - done() - }) - } + axios.get(`http://localhost:${port}/user`).catch(done) + }) - axios.get(`http://localhost:${port}/user`).catch(done) - }) + it(`should run the response's close event in the correct context`, done => { + app = (req, res) => { + const span = tracer.scope().active() - it(`should run the finish event in the correct context`, done => { - app = (req, res) => { - const span = tracer.scope().active() + res.on('close', () => { + expect(tracer.scope().active()).to.equal(span) + done() + }) + } - res.on('finish', () => { - expect(tracer.scope().active()).to.equal(span) - done() - }) - } + axios.get(`http://localhost:${port}/user`).catch(done) + }) - axios.get(`http://localhost:${port}/user`).catch(done) - }) + it(`should run the finish event in the correct context`, done => { + app = (req, res) => { + const span = tracer.scope().active() - it('should not instrument manually instantiated server responses', () => { - const { IncomingMessage, ServerResponse } = http + res.on('finish', () => { + expect(tracer.scope().active()).to.equal(span) + done() + }) + } - const req = new IncomingMessage() - const res = new ServerResponse(req) + axios.get(`http://localhost:${port}/user`).catch(done) + }) - expect(() => res.emit('finish')).to.not.throw() - }) + it('should not instrument manually instantiated server responses', () => { + const { IncomingMessage, ServerResponse } = http - it('should not cause `end` to be called multiple times', done => { - app = (req, res) => { - res.end = sinon.spy(res.end) + const req = new IncomingMessage() + const res = new ServerResponse(req) - res.on('finish', () => { - expect(res.end).to.have.been.calledOnce - done() - }) - } + expect(() => res.emit('finish')).to.not.throw() + }) - axios.get(`http://localhost:${port}/user`).catch(done) - }) - }) + it('should not cause `end` to be called multiple times', done => { + app = (req, res) => { + res.end = sinon.spy(res.end) - describe('with a `server` configuration', () => { - beforeEach(() => { - return agent.load('http', { client: false, server: {} }) - .then(() => { - http = require('http') - }) - }) + res.on('finish', () => { + expect(res.end).to.have.been.calledOnce + done() + }) + } - beforeEach(done => { - const server = new http.Server(listener) - appListener = server - .listen(port, 'localhost', () => done()) + axios.get(`http://localhost:${port}/user`).catch(done) + }) }) - // see https://github.com/DataDog/dd-trace-js/issues/2453 - it('should not have disabled tracing', (done) => { - agent.use(() => {}) - .then(done) - .catch(done) + describe('with a `server` configuration', () => { + beforeEach(() => { + return agent.load('http', { client: false, server: {} }) + .then(() => { + http = require(pluginToBeLoaded) + }) + }) - axios.get(`http://localhost:${port}/user`).catch(done) - }) - }) + beforeEach(done => { + const server = new http.Server(listener) + appListener = server + .listen(port, 'localhost', () => done()) + }) - describe('with a blocklist configuration', () => { - beforeEach(() => { - return agent.load('http', { client: false, blocklist: '/health' }) - .then(() => { - http = require('http') - }) - }) + // see https://github.com/DataDog/dd-trace-js/issues/2453 + it('should not have disabled tracing', (done) => { + agent.use(() => {}) + .then(done) + .catch(done) - beforeEach(done => { - const server = new http.Server(listener) - appListener = server - .listen(port, 'localhost', () => done()) + axios.get(`http://localhost:${port}/user`).catch(done) + }) }) - it('should drop traces for blocklist route', done => { - const spy = sinon.spy(() => {}) + describe('with a blocklist configuration', () => { + beforeEach(() => { + return agent.load('http', { client: false, blocklist: '/health' }) + .then(() => { + http = require(pluginToBeLoaded) + }) + }) - agent - .use((traces) => { - spy() - }) - .catch(done) + beforeEach(done => { + const server = new http.Server(listener) + appListener = server + .listen(port, 'localhost', () => done()) + }) - setTimeout(() => { - expect(spy).to.not.have.been.called - done() - }, 100) + it('should drop traces for blocklist route', done => { + const spy = sinon.spy(() => {}) - axios.get(`http://localhost:${port}/health`).catch(done) + agent + .use((traces) => { + spy() + }) + .catch(done) + + setTimeout(() => { + expect(spy).to.not.have.been.called + done() + }, 100) + + axios.get(`http://localhost:${port}/health`).catch(done) + }) }) }) }) diff --git a/packages/datadog-plugin-http2/test/client.spec.js b/packages/datadog-plugin-http2/test/client.spec.js index 89ec4cb1ab3..7877be9c427 100644 --- a/packages/datadog-plugin-http2/test/client.spec.js +++ b/packages/datadog-plugin-http2/test/client.spec.js @@ -20,15 +20,17 @@ describe('Plugin', () => { let appListener let tracer - ['http', 'https'].forEach(protocol => { - describe(`http2/client, protocol ${protocol}`, () => { + ['http', 'https', 'node:http', 'node:https'].forEach(pluginToBeLoaded => { + const protocol = pluginToBeLoaded.split(':')[1] || pluginToBeLoaded + const loadPlugin = pluginToBeLoaded.includes('node:') ? 'node:http2' : 'http2' + describe(`http2/client, protocol ${pluginToBeLoaded}`, () => { function server (app, port, listener) { let server - if (protocol === 'https') { + if (pluginToBeLoaded === 'https' || pluginToBeLoaded === 'node:https') { process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0' - server = require('http2').createSecureServer({ key, cert }) + server = require(loadPlugin).createSecureServer({ key, cert }) } else { - server = require('http2').createServer() + server = require(loadPlugin).createServer() } server.on('stream', app) server.listen(port, 'localhost', listener) @@ -51,7 +53,7 @@ describe('Plugin', () => { beforeEach(() => { return agent.load('http2', { server: false }) .then(() => { - http2 = require('http2') + http2 = require(loadPlugin) }) }) @@ -629,7 +631,7 @@ describe('Plugin', () => { }) it('should only record a request once', done => { - require('http2') + require(loadPlugin) const app = (stream, headers) => { stream.respond({ ':status': 200 @@ -682,7 +684,7 @@ describe('Plugin', () => { return agent.load('http2', config) .then(() => { - http2 = require('http2') + http2 = require(loadPlugin) }) }) @@ -729,7 +731,7 @@ describe('Plugin', () => { return agent.load('http2', config) .then(() => { - http2 = require('http2') + http2 = require(loadPlugin) }) }) @@ -777,7 +779,7 @@ describe('Plugin', () => { return agent.load('http2', config) .then(() => { - http2 = require('http2') + http2 = require(loadPlugin) }) }) @@ -856,7 +858,7 @@ describe('Plugin', () => { return agent.load('http2', config) .then(() => { - http2 = require('http2') + http2 = require(loadPlugin) }) }) @@ -905,7 +907,7 @@ describe('Plugin', () => { return agent.load('http2', config) .then(() => { - http2 = require('http2') + http2 = require(loadPlugin) }) }) diff --git a/packages/datadog-plugin-http2/test/server.spec.js b/packages/datadog-plugin-http2/test/server.spec.js index 47e54c2a29e..2c2be7175bb 100644 --- a/packages/datadog-plugin-http2/test/server.spec.js +++ b/packages/datadog-plugin-http2/test/server.spec.js @@ -51,201 +51,203 @@ describe('Plugin', () => { let port let app - describe('http2/server', () => { - beforeEach(() => { - tracer = require('../../dd-trace') - listener = (req, res) => { - app && app(req, res) - res.writeHead(200) - res.end() - } - }) - - beforeEach(() => { - return getPort().then(newPort => { - port = newPort - }) - }) - - afterEach(() => { - appListener && appListener.close() - app = null - return agent.close({ ritmReset: false }) - }) - - describe('cancelled request', () => { + ['http2', 'node:http2'].forEach(pluginToBeLoaded => { + describe(`${pluginToBeLoaded}/server`, () => { beforeEach(() => { + tracer = require('../../dd-trace') listener = (req, res) => { - setTimeout(() => { - app && app(req, res) - res.writeHead(200) - res.end() - }, 500) + app && app(req, res) + res.writeHead(200) + res.end() } }) beforeEach(() => { - return agent.load('http2') - .then(() => { - http2 = require('http2') - }) + return getPort().then(newPort => { + port = newPort + }) }) - beforeEach(done => { - const server = http2.createServer(listener) - appListener = server - .listen(port, 'localhost', () => done()) + afterEach(() => { + appListener && appListener.close() + app = null + return agent.close({ ritmReset: false }) }) - it('should send traces to agent', (done) => { - app = sinon.stub() - agent - .use(traces => { - expect(app).not.to.have.been.called // request should be cancelled before call to app - expect(traces[0][0]).to.have.property('name', 'web.request') - expect(traces[0][0]).to.have.property('service', 'test') - expect(traces[0][0]).to.have.property('type', 'web') - expect(traces[0][0]).to.have.property('resource', 'GET') - expect(traces[0][0].meta).to.have.property('span.kind', 'server') - expect(traces[0][0].meta).to.have.property('http.url', `http://localhost:${port}/user`) - expect(traces[0][0].meta).to.have.property('http.method', 'GET') - expect(traces[0][0].meta).to.have.property('http.status_code', '200') - expect(traces[0][0].meta).to.have.property('component', 'http2') - }) - .then(done) - .catch(done) + describe('cancelled request', () => { + beforeEach(() => { + listener = (req, res) => { + setTimeout(() => { + app && app(req, res) + res.writeHead(200) + res.end() + }, 500) + } + }) - // Don't use real AbortController because it requires 15.x+ - const ac = new MockAbortController() - request(http2, `http://localhost:${port}/user`, { - signal: ac.signal + beforeEach(() => { + return agent.load('http2') + .then(() => { + http2 = require(pluginToBeLoaded) + }) }) - setTimeout(() => { ac.abort() }, 100) - }) - }) - describe('without configuration', () => { - beforeEach(() => { - return agent.load('http2') - .then(() => { - http2 = require('http2') - }) - }) + beforeEach(done => { + const server = http2.createServer(listener) + appListener = server + .listen(port, 'localhost', () => done()) + }) - beforeEach(done => { - const server = http2.createServer(listener) - appListener = server - .listen(port, 'localhost', () => done()) + it('should send traces to agent', (done) => { + app = sinon.stub() + agent + .use(traces => { + expect(app).not.to.have.been.called // request should be cancelled before call to app + expect(traces[0][0]).to.have.property('name', 'web.request') + expect(traces[0][0]).to.have.property('service', 'test') + expect(traces[0][0]).to.have.property('type', 'web') + expect(traces[0][0]).to.have.property('resource', 'GET') + expect(traces[0][0].meta).to.have.property('span.kind', 'server') + expect(traces[0][0].meta).to.have.property('http.url', `http://localhost:${port}/user`) + expect(traces[0][0].meta).to.have.property('http.method', 'GET') + expect(traces[0][0].meta).to.have.property('http.status_code', '200') + expect(traces[0][0].meta).to.have.property('component', 'http2') + }) + .then(done) + .catch(done) + + // Don't use real AbortController because it requires 15.x+ + const ac = new MockAbortController() + request(http2, `http://localhost:${port}/user`, { + signal: ac.signal + }) + setTimeout(() => { ac.abort() }, 100) + }) }) - const spanProducerFn = (done) => { - request(http2, `http://localhost:${port}/user`).catch(done) - } - - withNamingSchema( - spanProducerFn, - rawExpectedSchema.server - ) - - it('should do automatic instrumentation', done => { - agent - .use(traces => { - expect(traces[0][0]).to.have.property('name', 'web.request') - expect(traces[0][0]).to.have.property('service', 'test') - expect(traces[0][0]).to.have.property('type', 'web') - expect(traces[0][0]).to.have.property('resource', 'GET') - expect(traces[0][0].meta).to.have.property('span.kind', 'server') - expect(traces[0][0].meta).to.have.property('http.url', `http://localhost:${port}/user`) - expect(traces[0][0].meta).to.have.property('http.method', 'GET') - expect(traces[0][0].meta).to.have.property('http.status_code', '200') - expect(traces[0][0].meta).to.have.property('component', 'http2') - }) - .then(done) - .catch(done) + describe('without configuration', () => { + beforeEach(() => { + return agent.load('http2') + .then(() => { + http2 = require(pluginToBeLoaded) + }) + }) - request(http2, `http://localhost:${port}/user`).catch(done) - }) + beforeEach(done => { + const server = http2.createServer(listener) + appListener = server + .listen(port, 'localhost', () => done()) + }) - it(`should run the request's close event in the correct context`, done => { - app = (req, res) => { - req.on('close', () => { - expect(tracer.scope().active()).to.equal(null) - done() - }) + const spanProducerFn = (done) => { + request(http2, `http://localhost:${port}/user`).catch(done) } - request(http2, `http://localhost:${port}/user`).catch(done) - }) + withNamingSchema( + spanProducerFn, + rawExpectedSchema.server + ) + + it('should do automatic instrumentation', done => { + agent + .use(traces => { + expect(traces[0][0]).to.have.property('name', 'web.request') + expect(traces[0][0]).to.have.property('service', 'test') + expect(traces[0][0]).to.have.property('type', 'web') + expect(traces[0][0]).to.have.property('resource', 'GET') + expect(traces[0][0].meta).to.have.property('span.kind', 'server') + expect(traces[0][0].meta).to.have.property('http.url', `http://localhost:${port}/user`) + expect(traces[0][0].meta).to.have.property('http.method', 'GET') + expect(traces[0][0].meta).to.have.property('http.status_code', '200') + expect(traces[0][0].meta).to.have.property('component', 'http2') + }) + .then(done) + .catch(done) + + request(http2, `http://localhost:${port}/user`).catch(done) + }) - it(`should run the response's close event in the correct context`, done => { - app = (req, res) => { - const span = tracer.scope().active() + it(`should run the request's close event in the correct context`, done => { + app = (req, res) => { + req.on('close', () => { + expect(tracer.scope().active()).to.equal(null) + done() + }) + } - res.on('close', () => { - expect(tracer.scope().active()).to.equal(span) - done() - }) - } + request(http2, `http://localhost:${port}/user`).catch(done) + }) - request(http2, `http://localhost:${port}/user`).catch(done) - }) + it(`should run the response's close event in the correct context`, done => { + app = (req, res) => { + const span = tracer.scope().active() - it(`should run the finish event in the correct context`, done => { - app = (req, res) => { - const span = tracer.scope().active() + res.on('close', () => { + expect(tracer.scope().active()).to.equal(span) + done() + }) + } - res.on('finish', () => { - expect(tracer.scope().active()).to.equal(span) - done() - }) - } + request(http2, `http://localhost:${port}/user`).catch(done) + }) - request(http2, `http://localhost:${port}/user`).catch(done) - }) + it(`should run the finish event in the correct context`, done => { + app = (req, res) => { + const span = tracer.scope().active() - it('should not cause `end` to be called multiple times', done => { - app = (req, res) => { - res.end = sinon.spy(res.end) + res.on('finish', () => { + expect(tracer.scope().active()).to.equal(span) + done() + }) + } - res.on('finish', () => { - expect(res.end).to.have.been.calledOnce - done() - }) - } + request(http2, `http://localhost:${port}/user`).catch(done) + }) - request(http2, `http://localhost:${port}/user`).catch(done) - }) - }) + it('should not cause `end` to be called multiple times', done => { + app = (req, res) => { + res.end = sinon.spy(res.end) - describe('with a blocklist configuration', () => { - beforeEach(() => { - return agent.load('http2', { client: false, blocklist: '/health' }) - .then(() => { - http2 = require('http2') - }) - }) + res.on('finish', () => { + expect(res.end).to.have.been.calledOnce + done() + }) + } - beforeEach(done => { - const server = http2.createServer(listener) - appListener = server - .listen(port, 'localhost', () => done()) + request(http2, `http://localhost:${port}/user`).catch(done) + }) }) - it('should drop traces for blocklist route', done => { - const spy = sinon.spy(() => {}) + describe('with a blocklist configuration', () => { + beforeEach(() => { + return agent.load('http2', { client: false, blocklist: '/health' }) + .then(() => { + http2 = require(pluginToBeLoaded) + }) + }) + + beforeEach(done => { + const server = http2.createServer(listener) + appListener = server + .listen(port, 'localhost', () => done()) + }) - agent - .use((traces) => { - spy() - }) - .catch(done) + it('should drop traces for blocklist route', done => { + const spy = sinon.spy(() => {}) - setTimeout(() => { - expect(spy).to.not.have.been.called - done() - }, 100) + agent + .use((traces) => { + spy() + }) + .catch(done) - request(http2, `http://localhost:${port}/health`).catch(done) + setTimeout(() => { + expect(spy).to.not.have.been.called + done() + }, 100) + + request(http2, `http://localhost:${port}/health`).catch(done) + }) }) }) }) diff --git a/packages/datadog-plugin-net/test/index.spec.js b/packages/datadog-plugin-net/test/index.spec.js index 7f1cf5d3e33..09532ad0502 100644 --- a/packages/datadog-plugin-net/test/index.spec.js +++ b/packages/datadog-plugin-net/test/index.spec.js @@ -15,274 +15,290 @@ describe('Plugin', () => { let tracer let parent - describe('net', () => { - afterEach(() => { - return agent.close() - }) - - afterEach(() => { - tcp.close() - }) + ['net', 'node:net'].forEach(pluginToBeLoaded => { + describe(pluginToBeLoaded, () => { + afterEach(() => { + return agent.close() + }) - afterEach(() => { - ipc.close() - }) + afterEach(() => { + tcp.close() + }) - beforeEach(() => { - return agent.load('net') - .then(() => { - net = require(`net`) - tracer = require('../../dd-trace') - parent = tracer.startSpan('parent') - parent.finish() + afterEach(() => { + ipc.close() + }) - return getPort() - }).then(_port => { - port = _port + beforeEach(() => { + return agent.load(['net', 'dns']) + .then(() => { + net = require(pluginToBeLoaded) + tracer = require('../../dd-trace') + parent = tracer.startSpan('parent') + parent.finish() - return new Promise(resolve => setImmediate(resolve)) - }) - }) + return getPort() + }).then(_port => { + port = _port - beforeEach(done => { - tcp = new net.Server(socket => { - socket.write('') + return new Promise(resolve => setImmediate(resolve)) + }) }) - tcp.listen(port, () => done()) - }) - beforeEach(done => { - ipc = new net.Server(socket => { - socket.write('') + beforeEach(done => { + tcp = new net.Server(socket => { + socket.write('') + }) + tcp.listen(port, () => done()) }) - ipc.listen('/tmp/dd-trace.sock', () => done()) - }) - it('should instrument connect with a path', done => { - expectSomeSpan(agent, { - name: 'ipc.connect', - service: 'test', - resource: '/tmp/dd-trace.sock', - meta: { - 'span.kind': 'client', - 'ipc.path': '/tmp/dd-trace.sock' - }, - parent_id: new Int64BE(parent.context()._spanId._buffer) - }).then(done).catch(done) - - tracer.scope().activate(parent, () => { - net.connect('/tmp/dd-trace.sock') + beforeEach(done => { + ipc = new net.Server(socket => { + socket.write('') + }) + ipc.listen('/tmp/dd-trace.sock', () => done()) }) - }) - withPeerService( - () => tracer, - 'net', - () => { - const socket = new net.Socket() - socket.connect(port, 'localhost') - }, - 'localhost', - 'out.host' - ) - - it('should instrument connect with a port', done => { - const socket = new net.Socket() - tracer.scope().activate(parent, () => { - socket.connect(port, 'localhost') - socket.on('connect', () => { - expectSomeSpan(agent, { - name: 'tcp.connect', - service: 'test', - resource: `localhost:${port}`, - meta: { - 'component': 'net', - 'span.kind': 'client', - 'tcp.family': 'IPv4', - 'tcp.remote.host': 'localhost', - 'tcp.local.address': socket.localAddress, - 'out.host': 'localhost' - }, - metrics: { - 'network.destination.port': port, - 'tcp.remote.port': port, - 'tcp.local.port': socket.localPort - }, - parent_id: new Int64BE(parent.context()._spanId._buffer) - }, 2000).then(done).catch(done) + it('should instrument connect with a path', done => { + expectSomeSpan(agent, { + name: 'ipc.connect', + service: 'test', + resource: '/tmp/dd-trace.sock', + meta: { + 'span.kind': 'client', + 'ipc.path': '/tmp/dd-trace.sock' + }, + parent_id: new Int64BE(parent.context()._spanId._buffer) + }).then(done).catch(done) + + tracer.scope().activate(parent, () => { + net.connect('/tmp/dd-trace.sock') }) }) - }) - it('should instrument connect with TCP options', done => { - const socket = new net.Socket() - tracer.scope().activate(parent, () => { - socket.connect({ - port, - host: 'localhost' - }) - socket.on('connect', () => { - expectSomeSpan(agent, { - name: 'tcp.connect', - service: 'test', - resource: `localhost:${port}`, - meta: { - 'component': 'net', - 'span.kind': 'client', - 'tcp.family': 'IPv4', - 'tcp.remote.host': 'localhost', - 'tcp.local.address': socket.localAddress, - 'out.host': 'localhost' - }, - metrics: { - 'network.destination.port': port, - 'tcp.remote.port': port, - 'tcp.local.port': socket.localPort - }, - parent_id: new Int64BE(parent.context()._spanId._buffer) - }).then(done).catch(done) + it('should instrument dns', done => { + const socket = new net.Socket() + tracer.scope().activate(parent, () => { + socket.connect(port, 'localhost') + socket.on('connect', () => { + expectSomeSpan(agent, { + name: 'dns.lookup', + service: 'test', + resource: 'localhost' + }, 2000).then(done).catch(done) + }) }) }) - }) - it('should instrument connect with IPC options', done => { - expectSomeSpan(agent, { - name: 'ipc.connect', - service: 'test', - resource: '/tmp/dd-trace.sock', - meta: { - 'component': 'net', - 'span.kind': 'client', - 'ipc.path': '/tmp/dd-trace.sock' + withPeerService( + () => tracer, + 'net', + () => { + const socket = new net.Socket() + socket.connect(port, 'localhost') }, - parent_id: new Int64BE(parent.context()._spanId._buffer) - }).then(done).catch(done) + 'localhost', + 'out.host' + ) - tracer.scope().activate(parent, () => { - net.connect({ - path: '/tmp/dd-trace.sock' + it('should instrument connect with a port', done => { + const socket = new net.Socket() + tracer.scope().activate(parent, () => { + socket.connect(port, 'localhost') + socket.on('connect', () => { + expectSomeSpan(agent, { + name: 'tcp.connect', + service: 'test', + resource: `localhost:${port}`, + meta: { + 'component': 'net', + 'span.kind': 'client', + 'tcp.family': 'IPv4', + 'tcp.remote.host': 'localhost', + 'tcp.local.address': socket.localAddress, + 'out.host': 'localhost' + }, + metrics: { + 'network.destination.port': port, + 'tcp.remote.port': port, + 'tcp.local.port': socket.localPort + }, + parent_id: new Int64BE(parent.context()._spanId._buffer) + }, 2000).then(done).catch(done) + }) }) }) - }) - - it('should instrument error', done => { - const socket = new net.Socket() - - let error = null - agent - .use(traces => { - expect(traces[0][0]).to.deep.include({ - name: 'tcp.connect', - service: 'test', - resource: `localhost:${port}` - }) - expect(traces[0][0].meta).to.deep.include({ - 'component': 'net', - 'span.kind': 'client', - 'tcp.family': 'IPv4', - 'tcp.remote.host': 'localhost', - 'out.host': 'localhost', - [ERROR_TYPE]: error.name, - [ERROR_MESSAGE]: error.message || error.code, - [ERROR_STACK]: error.stack + it('should instrument connect with TCP options', done => { + const socket = new net.Socket() + tracer.scope().activate(parent, () => { + socket.connect({ + port, + host: 'localhost' }) - expect(traces[0][0].metrics).to.deep.include({ - 'network.destination.port': port, - 'tcp.remote.port': port + socket.on('connect', () => { + expectSomeSpan(agent, { + name: 'tcp.connect', + service: 'test', + resource: `localhost:${port}`, + meta: { + 'component': 'net', + 'span.kind': 'client', + 'tcp.family': 'IPv4', + 'tcp.remote.host': 'localhost', + 'tcp.local.address': socket.localAddress, + 'out.host': 'localhost' + }, + metrics: { + 'network.destination.port': port, + 'tcp.remote.port': port, + 'tcp.local.port': socket.localPort + }, + parent_id: new Int64BE(parent.context()._spanId._buffer) + }).then(done).catch(done) }) - expect(traces[0][0].parent_id.toString()).to.equal(parent.context().toSpanId()) }) - .then(done) - .catch(done) + }) - tracer.scope().activate(parent, () => { - tcp.close() - socket.connect({ port }) - socket.once('error', (err) => { - error = err + it('should instrument connect with IPC options', done => { + expectSomeSpan(agent, { + name: 'ipc.connect', + service: 'test', + resource: '/tmp/dd-trace.sock', + meta: { + 'component': 'net', + 'span.kind': 'client', + 'ipc.path': '/tmp/dd-trace.sock' + }, + parent_id: new Int64BE(parent.context()._spanId._buffer) + }).then(done).catch(done) + + tracer.scope().activate(parent, () => { + net.connect({ + path: '/tmp/dd-trace.sock' + }) }) }) - }) - - it('should cleanup event listeners when the socket changes state', done => { - const socket = new net.Socket() - tracer.scope().activate(parent, () => { - const events = ['connect', 'error', 'close', 'timeout'] + it('should instrument error', done => { + const socket = new net.Socket() - socket.connect({ port }) - socket.destroy() + let error = null - socket.once('close', () => { - expect(socket.eventNames()).to.not.include.members(events) - done() + agent + .use(traces => { + expect(traces[0][0]).to.deep.include({ + name: 'tcp.connect', + service: 'test', + resource: `localhost:${port}` + }) + expect(traces[0][0].meta).to.deep.include({ + 'component': 'net', + 'span.kind': 'client', + 'tcp.family': 'IPv4', + 'tcp.remote.host': 'localhost', + 'out.host': 'localhost', + [ERROR_TYPE]: error.name, + [ERROR_MESSAGE]: error.message || error.code, + [ERROR_STACK]: error.stack + }) + expect(traces[0][0].metrics).to.deep.include({ + 'network.destination.port': port, + 'tcp.remote.port': port + }) + expect(traces[0][0].parent_id.toString()).to.equal(parent.context().toSpanId()) + }) + .then(done) + .catch(done) + + tracer.scope().activate(parent, () => { + tcp.close() + socket.connect({ port }) + socket.once('error', (err) => { + error = err + }) }) }) - }) - it('should run event listeners in the correct scope', () => { - return tracer.scope().activate(parent, () => { + it('should cleanup event listeners when the socket changes state', done => { const socket = new net.Socket() - const promises = Array(5).fill(0).map(() => { - let res - let rej - const p = new Promise((resolve, reject) => { - res = resolve - rej = reject + tracer.scope().activate(parent, () => { + const events = ['connect', 'error', 'close', 'timeout'] + + socket.connect({ port }) + socket.destroy() + + socket.once('close', () => { + expect(socket.eventNames()).to.not.include.members(events) + done() }) - p.resolve = res - p.reject = rej - return p }) + }) - socket.on('connect', () => { - expect(tracer.scope().active()).to.equal(parent) - promises[0].resolve() - }) + it('should run event listeners in the correct scope', () => { + return tracer.scope().activate(parent, () => { + const socket = new net.Socket() + + const promises = Array(5).fill(0).map(() => { + let res + let rej + const p = new Promise((resolve, reject) => { + res = resolve + rej = reject + }) + p.resolve = res + p.reject = rej + return p + }) - socket.on('ready', () => { - expect(tracer.scope().active()).to.equal(parent) - socket.destroy() - promises[1].resolve() - }) + socket.on('connect', () => { + expect(tracer.scope().active()).to.equal(parent) + promises[0].resolve() + }) - socket.on('close', () => { - expect(tracer.scope().active()).to.not.be.null - expect(tracer.scope().active().context()._name).to.equal('tcp.connect') - promises[2].resolve() - }) + socket.on('ready', () => { + expect(tracer.scope().active()).to.equal(parent) + socket.destroy() + promises[1].resolve() + }) - socket.on('lookup', () => { - expect(tracer.scope().active()).to.not.be.null - expect(tracer.scope().active().context()._name).to.equal('tcp.connect') - promises[3].resolve() - }) + socket.on('close', () => { + expect(tracer.scope().active()).to.not.be.null + expect(tracer.scope().active().context()._name).to.equal('tcp.connect') + promises[2].resolve() + }) - socket.connect({ - port, - lookup: (...args) => { + socket.on('lookup', () => { expect(tracer.scope().active()).to.not.be.null expect(tracer.scope().active().context()._name).to.equal('tcp.connect') - promises[4].resolve() - dns.lookup(...args) - } - }) + promises[3].resolve() + }) + + socket.connect({ + port, + lookup: (...args) => { + expect(tracer.scope().active()).to.not.be.null + expect(tracer.scope().active().context()._name).to.equal('tcp.connect') + promises[4].resolve() + dns.lookup(...args) + } + }) - return Promise.all(promises) + return Promise.all(promises) + }) }) - }) - it('should run the connection callback in the correct scope', done => { - const socket = new net.Socket() + it('should run the connection callback in the correct scope', done => { + const socket = new net.Socket() - tracer.scope().activate(parent, () => { - socket.connect({ port }, function () { - expect(this).to.equal(socket) - expect(tracer.scope().active()).to.equal(parent) - socket.destroy() - done() + tracer.scope().activate(parent, () => { + socket.connect({ port }, function () { + expect(this).to.equal(socket) + expect(tracer.scope().active()).to.equal(parent) + socket.destroy() + done() + }) }) }) }) diff --git a/packages/dd-trace/src/plugins/index.js b/packages/dd-trace/src/plugins/index.js index d2a22cd8b15..c7c96df0f50 100644 --- a/packages/dd-trace/src/plugins/index.js +++ b/packages/dd-trace/src/plugins/index.js @@ -59,6 +59,11 @@ module.exports = { get 'mysql2' () { return require('../../../datadog-plugin-mysql2/src') }, get 'net' () { return require('../../../datadog-plugin-net/src') }, get 'next' () { return require('../../../datadog-plugin-next/src') }, + get 'node:dns' () { return require('../../../datadog-plugin-dns/src') }, + get 'node:http' () { return require('../../../datadog-plugin-http/src') }, + get 'node:http2' () { return require('../../../datadog-plugin-http2/src') }, + get 'node:https' () { return require('../../../datadog-plugin-http/src') }, + get 'node:net' () { return require('../../../datadog-plugin-net/src') }, get 'oracledb' () { return require('../../../datadog-plugin-oracledb/src') }, get 'openai' () { return require('../../../datadog-plugin-openai/src') }, get 'paperplane' () { return require('../../../datadog-plugin-paperplane/src') }, From 9d611ad5c9f4f2965b44034671074ce3f1ac1fd3 Mon Sep 17 00:00:00 2001 From: Ayan Khan Date: Wed, 27 Dec 2023 15:22:47 -0500 Subject: [PATCH 06/21] simplify ci build for aerospike (#3886) * simplify ci build for aerospike --- .github/workflows/plugins.yml | 55 +++++++++++++++-------------------- 1 file changed, 24 insertions(+), 31 deletions(-) diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml index 59f2dcd3934..7c6f1f10280 100644 --- a/.github/workflows/plugins.yml +++ b/.github/workflows/plugins.yml @@ -59,11 +59,7 @@ jobs: majorVersion=$(echo "$version" | cut -d '.' -f 1) echo "Major Version: $majorVersion" echo "MAJOR_VERSION=$majorVersion" >> $GITHUB_ENV - - name: Check package version - if: env.MAJOR_VERSION == '3' - run: | - echo "Package version is 3. Proceeding with the next steps." - - name: Install dependencies + - name: Install dependencies and run tests if: env.MAJOR_VERSION == '3' run: | apt-get update && \ @@ -72,14 +68,11 @@ jobs: wget \ g++ libssl1.0.0 libssl-dev zlib1g-dev && \ npm install -g yarn - - if: env.MAJOR_VERSION == '3' - run: yarn install --ignore-engines - - if: env.MAJOR_VERSION == '3' - uses: ./.github/actions/node/14 - - if: env.MAJOR_VERSION == '3' - run: yarn test:plugins:ci - - if: env.MAJOR_VERSION == '3' + yarn install --ignore-engines + yarn test:plugins:ci + - if: always() uses: codecov/codecov-action@v2 + aerospike-4: runs-on: ubuntu-latest services: @@ -101,7 +94,15 @@ jobs: - if: always() uses: ./.github/actions/testagent/logs - uses: codecov/codecov-action@v2 + aerospike-5: + strategy: + matrix: + node-version: [16] + range: ['5.5.0 - 5.7.0'] + include: + - node-version: 20 + range: '>=5.8.0' runs-on: ubuntu-latest services: aerospike: @@ -111,7 +112,7 @@ jobs: env: PLUGINS: aerospike SERVICES: aerospike - PACKAGE_VERSION_RANGE: '5.5.0 - 5.7.0' + PACKAGE_VERSION_RANGE: ${{ matrix.range }} steps: - uses: actions/checkout@v2 - uses: ./.github/actions/testagent/start @@ -126,26 +127,18 @@ jobs: majorVersion=$(echo "$version" | cut -d '.' -f 1) echo "Major Version: $majorVersion" echo "MAJOR_VERSION=$majorVersion" >> $GITHUB_ENV - - name: Check package version + - uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + - name: Install dependencies and run tests if: env.MAJOR_VERSION != '3' run: | - echo "Package version is not 3. Proceeding with the next steps." - - if: env.MAJOR_VERSION != '3' - run: yarn install --ignore-engines - - if: env.MAJOR_VERSION != '3' - uses: ./.github/actions/node/oldest - - if: env.MAJOR_VERSION != '3' - run: yarn test:plugins:ci - - if: env.MAJOR_VERSION != '3' - run: echo "PACKAGE_VERSION_RANGE=>=5.8.0" >> "$GITHUB_ENV" - - if: env.MAJOR_VERSION != '3' - uses: ./.github/actions/node/20 # currently the latest version of aerospike only supports node 20 - - if: env.MAJOR_VERSION != '3' - run: yarn test:plugins:ci - - if: env.MAJOR_VERSION != '3' + yarn install --ignore-engines + yarn test:plugins:ci + - if: always() uses: ./.github/actions/testagent/logs - - if: env.MAJOR_VERSION != '3' - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v2 + amqp10: # TODO: move rhea to its own job runs-on: ubuntu-latest services: @@ -1247,4 +1240,4 @@ jobs: - uses: ./.github/actions/node/latest - run: yarn test:plugins:ci - if: always() - uses: ./.github/actions/testagent/logs + uses: ./.github/actions/testagent/logs \ No newline at end of file From 02412e338e6c686a33cddd8dc21ba32bdc0a1d7b Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Thu, 28 Dec 2023 22:35:06 +0800 Subject: [PATCH 07/21] Fix net plugin tests (#3906) --- packages/datadog-plugin-net/test/index.spec.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/datadog-plugin-net/test/index.spec.js b/packages/datadog-plugin-net/test/index.spec.js index 09532ad0502..0f47b0c9034 100644 --- a/packages/datadog-plugin-net/test/index.spec.js +++ b/packages/datadog-plugin-net/test/index.spec.js @@ -230,8 +230,14 @@ describe('Plugin', () => { socket.destroy() socket.once('close', () => { - expect(socket.eventNames()).to.not.include.members(events) - done() + setImmediate(() => { + // Node.js 21.2 broke this function. We'll have to do the more manual way for now. + // expect(socket.eventNames()).to.not.include.members(events) + for (const event of events) { + expect(socket.listeners(event)).to.have.lengthOf(0) + } + done() + }) }) }) }) From fba890653afa6956f1af602f2a1502489b64241e Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Thu, 28 Dec 2023 22:47:28 +0800 Subject: [PATCH 08/21] Fix integration tests by pinned chai to v4 as v5 went ESM-only (#3909) --- integration-tests/ci-visibility.spec.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration-tests/ci-visibility.spec.js b/integration-tests/ci-visibility.spec.js index b9cf69c4c41..fa07ff4a29a 100644 --- a/integration-tests/ci-visibility.spec.js +++ b/integration-tests/ci-visibility.spec.js @@ -38,7 +38,7 @@ const mochaCommonOptions = { const jestCommonOptions = { name: 'jest', - dependencies: ['jest', 'chai', 'jest-jasmine2'], + dependencies: ['jest', 'chai@v4', 'jest-jasmine2'], expectedStdout: 'Test Suites: 2 passed', expectedCoverageFiles: [ 'ci-visibility/test/sum.js', @@ -51,7 +51,7 @@ const testFrameworks = [ { ...mochaCommonOptions, testFile: 'ci-visibility/run-mocha.js', - dependencies: ['mocha', 'chai', 'nyc'], + dependencies: ['mocha', 'chai@v4', 'nyc'], expectedCoverageFiles: [ 'ci-visibility/run-mocha.js', 'ci-visibility/test/sum.js', @@ -64,7 +64,7 @@ const testFrameworks = [ { ...mochaCommonOptions, testFile: 'ci-visibility/run-mocha.mjs', - dependencies: ['mocha', 'chai', 'nyc', '@istanbuljs/esm-loader-hook'], + dependencies: ['mocha', 'chai@v4', 'nyc', '@istanbuljs/esm-loader-hook'], expectedCoverageFiles: [ 'ci-visibility/run-mocha.mjs', 'ci-visibility/test/sum.js', From a57fe5b6b0bdd20cfa7cc66eb5fb42c4ec53028f Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Thu, 28 Dec 2023 23:23:01 +0800 Subject: [PATCH 09/21] Add install_signature to app-started telemetry event (#3903) --- packages/dd-trace/src/config.js | 19 +++++++++++++++++++ packages/dd-trace/src/telemetry/index.js | 15 +++++++++++++++ packages/dd-trace/test/config.spec.js | 11 +++++++++++ .../dd-trace/test/telemetry/index.spec.js | 15 ++++++++++++++- 4 files changed, 59 insertions(+), 1 deletion(-) diff --git a/packages/dd-trace/src/config.js b/packages/dd-trace/src/config.js index 40c9c68d091..1d5d4c17e7c 100644 --- a/packages/dd-trace/src/config.js +++ b/packages/dd-trace/src/config.js @@ -522,6 +522,19 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?) 0 ) + const DD_INSTRUMENTATION_INSTALL_ID = coalesce( + process.env.DD_INSTRUMENTATION_INSTALL_ID, + null + ) + const DD_INSTRUMENTATION_INSTALL_TIME = coalesce( + process.env.DD_INSTRUMENTATION_INSTALL_TIME, + null + ) + const DD_INSTRUMENTATION_INSTALL_TYPE = coalesce( + process.env.DD_INSTRUMENTATION_INSTALL_TYPE, + null + ) + const ingestion = options.ingestion || {} const dogstatsd = coalesce(options.dogstatsd, {}) const sampler = { @@ -671,6 +684,12 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?) this.spanLeakDebug = Number(DD_TRACE_SPAN_LEAK_DEBUG) + this.installSignature = { + id: DD_INSTRUMENTATION_INSTALL_ID, + time: DD_INSTRUMENTATION_INSTALL_TIME, + type: DD_INSTRUMENTATION_INSTALL_TYPE + } + this._applyDefaults() this._applyEnvironment() this._applyOptions(options) diff --git a/packages/dd-trace/src/telemetry/index.js b/packages/dd-trace/src/telemetry/index.js index f7a300309b6..a9a741b8539 100644 --- a/packages/dd-trace/src/telemetry/index.js +++ b/packages/dd-trace/src/telemetry/index.js @@ -112,11 +112,26 @@ function flatten (input, result = [], prefix = [], traversedObjects = null) { return result } +function getInstallSignature (config) { + const { installSignature: sig } = config + if (sig && (sig.id || sig.time || sig.type)) { + return { + install_id: sig.id, + install_time: sig.time, + install_type: sig.type + } + } +} + function appStarted (config) { const app = { products: getProducts(config), configuration: flatten(config) } + const installSignature = getInstallSignature(config) + if (installSignature) { + app.install_signature = installSignature + } // TODO: add app.error with correct error codes // if (errors.agentError) { // app.error = errors.agentError diff --git a/packages/dd-trace/test/config.spec.js b/packages/dd-trace/test/config.spec.js index 21047b5c894..6b14a307039 100644 --- a/packages/dd-trace/test/config.spec.js +++ b/packages/dd-trace/test/config.spec.js @@ -121,6 +121,9 @@ describe('Config', () => { expect(config).to.have.nested.property('iast.redactionNamePattern', null) expect(config).to.have.nested.property('iast.redactionValuePattern', null) expect(config).to.have.nested.property('iast.telemetryVerbosity', 'INFORMATION') + expect(config).to.have.nested.property('installSignature.id', null) + expect(config).to.have.nested.property('installSignature.time', null) + expect(config).to.have.nested.property('installSignature.type', null) }) it('should support logging', () => { @@ -229,6 +232,9 @@ describe('Config', () => { process.env.DD_EXPERIMENTAL_PROFILING_ENABLED = 'true' process.env.DD_EXPERIMENTAL_API_SECURITY_ENABLED = 'true' process.env.DD_API_SECURITY_REQUEST_SAMPLE_RATE = 1 + process.env.DD_INSTRUMENTATION_INSTALL_ID = '68e75c48-57ca-4a12-adfc-575c4b05fcbe' + process.env.DD_INSTRUMENTATION_INSTALL_TYPE = 'k8s_single_step' + process.env.DD_INSTRUMENTATION_INSTALL_TIME = '1703188212' const config = new Config() @@ -308,6 +314,11 @@ describe('Config', () => { expect(config).to.have.nested.property('iast.redactionNamePattern', 'REDACTION_NAME_PATTERN') expect(config).to.have.nested.property('iast.redactionValuePattern', 'REDACTION_VALUE_PATTERN') expect(config).to.have.nested.property('iast.telemetryVerbosity', 'DEBUG') + expect(config).to.have.deep.property('installSignature', { + id: '68e75c48-57ca-4a12-adfc-575c4b05fcbe', + type: 'k8s_single_step', + time: '1703188212' + }) }) it('should read case-insensitive booleans from environment variables', () => { diff --git a/packages/dd-trace/test/telemetry/index.spec.js b/packages/dd-trace/test/telemetry/index.spec.js index ea6f974d6d3..5bb42a8c7c3 100644 --- a/packages/dd-trace/test/telemetry/index.spec.js +++ b/packages/dd-trace/test/telemetry/index.spec.js @@ -74,6 +74,11 @@ describe('telemetry', () => { peerServiceMapping: { 'service_1': 'remapped_service_1', 'service_2': 'remapped_service_2' + }, + installSignature: { + id: '68e75c48-57ca-4a12-adfc-575c4b05fcbe', + type: 'k8s_single_step', + time: '1703188212' } }, { _pluginsByName: pluginsByName @@ -105,8 +110,16 @@ describe('telemetry', () => { { name: 'appsec.enabled', value: true, origin: 'unknown' }, { name: 'profiling.enabled', value: true, origin: 'unknown' }, { name: 'peerServiceMapping.service_1', value: 'remapped_service_1', origin: 'unknown' }, - { name: 'peerServiceMapping.service_2', value: 'remapped_service_2', origin: 'unknown' } + { name: 'peerServiceMapping.service_2', value: 'remapped_service_2', origin: 'unknown' }, + { name: 'installSignature.id', value: '68e75c48-57ca-4a12-adfc-575c4b05fcbe', origin: 'unknown' }, + { name: 'installSignature.type', value: 'k8s_single_step', origin: 'unknown' }, + { name: 'installSignature.time', value: '1703188212', origin: 'unknown' } ]) + expect(payload).to.have.property('install_signature').that.deep.equal({ + install_id: '68e75c48-57ca-4a12-adfc-575c4b05fcbe', + install_type: 'k8s_single_step', + install_time: '1703188212' + }) }) }) From f532ddf34d15e82e1a34b812c4c644118fcf4279 Mon Sep 17 00:00:00 2001 From: Nicolas Savoire Date: Thu, 28 Dec 2023 16:25:15 +0100 Subject: [PATCH 10/21] Fix compatibility with node < 14.18 (#3908) Node 14 versions prior to 14.18 do not support require statements with `node:` prefix. --- integration-tests/profiler.spec.js | 8 ++++---- integration-tests/profiler/nettest.js | 3 +-- packages/dd-trace/src/profiling/profilers/events.js | 2 +- packages/dd-trace/src/profiling/profilers/shared.js | 2 +- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/integration-tests/profiler.spec.js b/integration-tests/profiler.spec.js index 8be691fe592..76685bc30be 100644 --- a/integration-tests/profiler.spec.js +++ b/integration-tests/profiler.spec.js @@ -8,10 +8,10 @@ const childProcess = require('child_process') const { fork } = childProcess const path = require('path') const { assert } = require('chai') -const fs = require('node:fs/promises') -const fsync = require('node:fs') -const net = require('node:net') -const zlib = require('node:zlib') +const fs = require('fs/promises') +const fsync = require('fs') +const net = require('net') +const zlib = require('zlib') const { Profile } = require('pprof-format') const semver = require('semver') diff --git a/integration-tests/profiler/nettest.js b/integration-tests/profiler/nettest.js index b98bc7d55f3..e9f3002d6b0 100644 --- a/integration-tests/profiler/nettest.js +++ b/integration-tests/profiler/nettest.js @@ -1,5 +1,4 @@ -const net = require('node:net') -const process = require('node:process') +const net = require('net') async function streamToString (stream) { const chunks = [] diff --git a/packages/dd-trace/src/profiling/profilers/events.js b/packages/dd-trace/src/profiling/profilers/events.js index 5c743bb96b2..45ee6f94009 100644 --- a/packages/dd-trace/src/profiling/profilers/events.js +++ b/packages/dd-trace/src/profiling/profilers/events.js @@ -1,4 +1,4 @@ -const { performance, constants, PerformanceObserver } = require('node:perf_hooks') +const { performance, constants, PerformanceObserver } = require('perf_hooks') const { END_TIMESTAMP_LABEL } = require('./shared') const semver = require('semver') const { Function, Label, Line, Location, Profile, Sample, StringTable, ValueType } = require('pprof-format') diff --git a/packages/dd-trace/src/profiling/profilers/shared.js b/packages/dd-trace/src/profiling/profilers/shared.js index 4337a80ae29..31dc7b2ce34 100644 --- a/packages/dd-trace/src/profiling/profilers/shared.js +++ b/packages/dd-trace/src/profiling/profilers/shared.js @@ -1,6 +1,6 @@ 'use strict' -const { isMainThread, threadId } = require('node:worker_threads') +const { isMainThread, threadId } = require('worker_threads') const END_TIMESTAMP_LABEL = 'end_timestamp_ns' const THREAD_NAME_LABEL = 'thread name' From 407271327327b4c8e66e10f9fece57a9c53d4906 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Fri, 29 Dec 2023 00:27:15 +0800 Subject: [PATCH 11/21] Fix timeouts from aws-sdk kinesis tests (#3910) --- packages/datadog-plugin-aws-sdk/test/kinesis.spec.js | 2 +- packages/datadog-plugin-aws-sdk/test/kinesis_helpers.js | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/datadog-plugin-aws-sdk/test/kinesis.spec.js b/packages/datadog-plugin-aws-sdk/test/kinesis.spec.js index db8177370c0..d3b6221d65a 100644 --- a/packages/datadog-plugin-aws-sdk/test/kinesis.spec.js +++ b/packages/datadog-plugin-aws-sdk/test/kinesis.spec.js @@ -41,7 +41,7 @@ describe('Kinesis', () => { }, (err, res) => { if (err) return done(err) - helpers.waitForActiveStream(this, kinesis, done) + helpers.waitForActiveStream(kinesis, done) }) }) diff --git a/packages/datadog-plugin-aws-sdk/test/kinesis_helpers.js b/packages/datadog-plugin-aws-sdk/test/kinesis_helpers.js index f76e6119251..8f91daab67d 100644 --- a/packages/datadog-plugin-aws-sdk/test/kinesis_helpers.js +++ b/packages/datadog-plugin-aws-sdk/test/kinesis_helpers.js @@ -45,17 +45,15 @@ function putTestRecord (kinesis, data, cb) { }, cb) } -function waitForActiveStream (mocha, kinesis, cb) { +function waitForActiveStream (kinesis, cb) { kinesis.describeStream({ StreamName: 'MyStream' }, (err, data) => { if (err) { - mocha.timeout(2000) - return waitForActiveStream(mocha, kinesis, cb) + return waitForActiveStream(kinesis, cb) } if (data.StreamDescription.StreamStatus !== 'ACTIVE') { - mocha.timeout(2000) - return waitForActiveStream(mocha, kinesis, cb) + return waitForActiveStream(kinesis, cb) } cb() From 7025612be5c25c1a771eff0211dd121c3f8af5a6 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Sat, 30 Dec 2023 00:19:42 +0800 Subject: [PATCH 12/21] Update actions versions (#3907) * Bump github actions versions * Stop using deprecated set-output command * Fix incompatible GLIBC version in GHA --- .github/actions/testagent/start/action.yml | 2 +- .github/workflows/appsec.yml | 48 ++-- .../workflows/ci-visibility-performance.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/core.yml | 4 +- .github/workflows/lambda.yml | 4 +- .github/workflows/package-size.yml | 2 +- .github/workflows/plugins.yml | 230 +++++++++--------- .github/workflows/profiling.yml | 12 +- .github/workflows/project.yml | 10 +- .github/workflows/release-3.yml | 8 +- .github/workflows/release-dev.yml | 6 +- .github/workflows/release-latest.yml | 14 +- .github/workflows/release-proposal.yml | 2 +- .../workflows/serverless-integration-test.yml | 2 +- .github/workflows/serverless-performance.yml | 4 +- .github/workflows/system-tests.yml | 8 +- .github/workflows/test-k8s-lib-injection.yaml | 2 +- .github/workflows/tracing.yml | 12 +- 19 files changed, 188 insertions(+), 186 deletions(-) diff --git a/.github/actions/testagent/start/action.yml b/.github/actions/testagent/start/action.yml index e5865983986..6f59559648e 100644 --- a/.github/actions/testagent/start/action.yml +++ b/.github/actions/testagent/start/action.yml @@ -3,6 +3,6 @@ description: "Starts the APM Test Agent image with environment." runs: using: composite steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - run: docker-compose up -d testagent shell: bash diff --git a/.github/workflows/appsec.yml b/.github/workflows/appsec.yml index a0e22b28ff8..2a1e2440b56 100644 --- a/.github/workflows/appsec.yml +++ b/.github/workflows/appsec.yml @@ -15,16 +15,16 @@ jobs: macos: runs-on: macos-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - run: yarn install - run: yarn test:appsec:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 ubuntu: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - run: yarn install - uses: ./.github/actions/node/16 @@ -33,16 +33,16 @@ jobs: - run: yarn test:appsec:ci - uses: ./.github/actions/node/latest - run: yarn test:appsec:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 windows: runs-on: windows-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - run: yarn install - run: yarn test:appsec:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 ldapjs: runs-on: ubuntu-latest @@ -60,14 +60,14 @@ jobs: LDAP_USERS: 'user01,user02' LDAP_PASSWORDS: 'password1,password2' steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - run: yarn install - uses: ./.github/actions/node/oldest - run: yarn test:appsec:plugins:ci - uses: ./.github/actions/node/latest - run: yarn test:appsec:plugins:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 postgres: runs-on: ubuntu-latest @@ -83,7 +83,7 @@ jobs: PLUGINS: pg|knex SERVICES: postgres steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - run: yarn install - uses: ./.github/actions/node/oldest @@ -94,7 +94,7 @@ jobs: - run: yarn test:appsec:plugins:ci - uses: ./.github/actions/node/20 - run: yarn test:appsec:plugins:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 mysql: runs-on: ubuntu-latest @@ -110,7 +110,7 @@ jobs: PLUGINS: mysql|mysql2|sequelize SERVICES: mysql steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - run: yarn install - uses: ./.github/actions/node/16 @@ -119,35 +119,35 @@ jobs: - run: yarn test:appsec:plugins:ci - uses: ./.github/actions/node/20 - run: yarn test:appsec:plugins:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 express: runs-on: ubuntu-latest env: PLUGINS: express|body-parser|cookie-parser steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - run: yarn install - uses: ./.github/actions/node/oldest - run: yarn test:appsec:plugins:ci - uses: ./.github/actions/node/latest - run: yarn test:appsec:plugins:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 graphql: runs-on: ubuntu-latest env: PLUGINS: apollo-server|apollo-server-express|apollo-server-fastify|apollo-server-core steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - run: yarn install - uses: ./.github/actions/node/oldest - run: yarn test:appsec:plugins:ci - uses: ./.github/actions/node/latest - run: yarn test:appsec:plugins:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 mongodb-core: runs-on: ubuntu-latest @@ -160,14 +160,14 @@ jobs: PLUGINS: express-mongo-sanitize SERVICES: mongo steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - run: yarn install - uses: ./.github/actions/node/oldest - run: yarn test:appsec:plugins:ci - uses: ./.github/actions/node/latest - run: yarn test:appsec:plugins:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 mongoose: runs-on: ubuntu-latest @@ -180,21 +180,21 @@ jobs: PLUGINS: mongoose SERVICES: mongo steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - run: yarn install - uses: ./.github/actions/node/oldest - run: yarn test:appsec:plugins:ci - uses: ./.github/actions/node/latest - run: yarn test:appsec:plugins:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 sourcing: runs-on: ubuntu-latest env: PLUGINS: cookie steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - run: yarn install - uses: ./.github/actions/node/16 @@ -205,7 +205,7 @@ jobs: - run: yarn test:appsec:plugins:ci - uses: ./.github/actions/node/latest - run: yarn test:appsec:plugins:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 next: strategy: @@ -220,7 +220,7 @@ jobs: PLUGINS: next RANGE: ${{ matrix.range }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - uses: actions/setup-node@v3 @@ -230,4 +230,4 @@ jobs: - run: yarn test:appsec:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 diff --git a/.github/workflows/ci-visibility-performance.yml b/.github/workflows/ci-visibility-performance.yml index c399c9b3096..2a24980b4d5 100644 --- a/.github/workflows/ci-visibility-performance.yml +++ b/.github/workflows/ci-visibility-performance.yml @@ -19,7 +19,7 @@ jobs: env: ROBOT_CI_GITHUB_PERSONAL_ACCESS_TOKEN: ${{ secrets.ROBOT_CI_GITHUB_PERSONAL_ACCESS_TOKEN }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/18 - name: CI Visibility Performance Overhead Test run: yarn bench:e2e:ci-visibility diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index af37ccf7d90..51af025df84 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -34,7 +34,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index e8661d9652b..322725b0f34 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -15,11 +15,11 @@ jobs: shimmer: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - run: yarn install - uses: ./.github/actions/node/oldest - run: yarn test:shimmer:ci - uses: ./.github/actions/node/latest - run: yarn test:shimmer:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 diff --git a/.github/workflows/lambda.yml b/.github/workflows/lambda.yml index 2600cc157f0..f98b74914e5 100644 --- a/.github/workflows/lambda.yml +++ b/.github/workflows/lambda.yml @@ -15,7 +15,7 @@ jobs: ubuntu: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -29,4 +29,4 @@ jobs: - run: yarn test:lambda:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 diff --git a/.github/workflows/package-size.yml b/.github/workflows/package-size.yml index a29c22f29cb..4b2934a20d1 100644 --- a/.github/workflows/package-size.yml +++ b/.github/workflows/package-size.yml @@ -13,7 +13,7 @@ jobs: package-size-report: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Setup Node.js uses: actions/setup-node@v2 with: diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml index 7c6f1f10280..d76e3f55a3f 100644 --- a/.github/workflows/plugins.yml +++ b/.github/workflows/plugins.yml @@ -45,14 +45,15 @@ jobs: DD_TEST_AGENT_URL: http://testagent:9126 AEROSPIKE_HOST_ADDRESS: aerospike steps: - - uses: actions/checkout@v2 + # Needs to remain on v3 for now due to GLIBC version + - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: node-version: '14' - id: pkg run: | content=`cat ./package.json | tr '\n' ' '` - echo "::set-output name=json::$content" + echo "json=$content" >> $GITHUB_OUTPUT - id: extract run: | version="${{fromJson(steps.pkg.outputs.json).version}}" @@ -71,7 +72,7 @@ jobs: yarn install --ignore-engines yarn test:plugins:ci - if: always() - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v3 aerospike-4: runs-on: ubuntu-latest @@ -85,7 +86,7 @@ jobs: SERVICES: aerospike PACKAGE_VERSION_RANGE: '4.0.0 - 5.4.0' steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install --ignore-engines @@ -93,7 +94,7 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 aerospike-5: strategy: @@ -114,13 +115,13 @@ jobs: SERVICES: aerospike PACKAGE_VERSION_RANGE: ${{ matrix.range }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - id: pkg run: | content=`cat ./package.json | tr '\n' ' '` - echo "::set-output name=json::$content" + echo "json=$content" >> $GITHUB_OUTPUT - id: extract run: | version="${{fromJson(steps.pkg.outputs.json).version}}" @@ -137,7 +138,7 @@ jobs: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 amqp10: # TODO: move rhea to its own job runs-on: ubuntu-latest @@ -153,7 +154,7 @@ jobs: PLUGINS: amqp10|rhea SERVICES: qpid steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -165,7 +166,7 @@ jobs: - run: yarn test:plugins:upstream - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 amqplib: runs-on: ubuntu-latest @@ -178,7 +179,7 @@ jobs: PLUGINS: amqplib SERVICES: rabbitmq steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -190,7 +191,7 @@ jobs: - run: yarn test:plugins:upstream - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 aws-sdk: runs-on: ubuntu-latest @@ -228,7 +229,7 @@ jobs: PLUGINS: aws-sdk SERVICES: localstack localstack-legacy steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -238,14 +239,14 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 axios: runs-on: ubuntu-latest env: PLUGINS: axios steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -255,14 +256,14 @@ jobs: - run: yarn test:plugins:upstream - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 bluebird: runs-on: ubuntu-latest env: PLUGINS: bluebird steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -270,7 +271,7 @@ jobs: - run: yarn test:plugins:ci - uses: ./.github/actions/node/latest - run: yarn test:plugins:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 - if: always() uses: ./.github/actions/testagent/logs @@ -279,7 +280,7 @@ jobs: env: PLUGINS: bunyan steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -291,7 +292,7 @@ jobs: - run: yarn test:plugins:upstream - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 cassandra: runs-on: ubuntu-latest @@ -306,7 +307,7 @@ jobs: PLUGINS: cassandra-driver SERVICES: cassandra steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -316,7 +317,7 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 couchbase: runs-on: ubuntu-latest @@ -330,20 +331,20 @@ jobs: PLUGINS: couchbase SERVICES: couchbase steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install - uses: ./.github/actions/node/oldest - run: yarn test:plugins:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 connect: runs-on: ubuntu-latest env: PLUGINS: connect steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -355,14 +356,14 @@ jobs: - run: yarn test:plugins:upstream - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 cucumber: runs-on: ubuntu-latest env: PLUGINS: cucumber steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -372,7 +373,7 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 # TODO: fix performance issues and test more Node versions cypress: @@ -380,21 +381,21 @@ jobs: env: PLUGINS: cypress steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 dns: runs-on: ubuntu-latest env: PLUGINS: dns steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -408,7 +409,7 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 elasticsearch: runs-on: ubuntu-latest @@ -423,7 +424,7 @@ jobs: PLUGINS: elasticsearch SERVICES: elasticsearch steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -431,14 +432,14 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 express: runs-on: ubuntu-latest env: PLUGINS: express|body-parser|cookie-parser steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -448,14 +449,14 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 fastify: runs-on: ubuntu-latest env: PLUGINS: fastify steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -465,14 +466,14 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 fetch: runs-on: ubuntu-latest env: PLUGINS: fetch steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -482,14 +483,14 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 generic-pool: runs-on: ubuntu-latest env: PLUGINS: generic-pool steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -499,7 +500,7 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 google-cloud-pubsub: runs-on: ubuntu-latest @@ -512,7 +513,7 @@ jobs: PLUGINS: google-cloud-pubsub SERVICES: gpubsub steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -522,14 +523,14 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 graphql: runs-on: ubuntu-latest env: PLUGINS: graphql steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -541,14 +542,14 @@ jobs: - run: yarn test:plugins:upstream - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 grpc: runs-on: ubuntu-latest env: PLUGINS: grpc steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -558,14 +559,14 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 hapi: runs-on: ubuntu-latest env: PLUGINS: hapi steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -575,14 +576,14 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 http: runs-on: ubuntu-latest env: PLUGINS: http steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -596,14 +597,14 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 http2: runs-on: ubuntu-latest env: PLUGINS: http2 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -617,7 +618,7 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 # TODO: fix performance issues and test more Node versions jest: @@ -625,14 +626,14 @@ jobs: env: PLUGINS: jest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 kafkajs: runs-on: ubuntu-latest @@ -653,7 +654,7 @@ jobs: PLUGINS: kafkajs SERVICES: kafka steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -663,14 +664,14 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 knex: runs-on: ubuntu-latest env: PLUGINS: knex steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -680,14 +681,14 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 koa: runs-on: ubuntu-latest env: PLUGINS: koa steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -699,7 +700,7 @@ jobs: - run: yarn test:plugins:upstream - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 limitd-client: runs-on: ubuntu-latest @@ -716,7 +717,7 @@ jobs: PLUGINS: limitd-client SERVICES: limitd steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -726,7 +727,7 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 memcached: runs-on: ubuntu-latest @@ -739,7 +740,7 @@ jobs: PLUGINS: memcached SERVICES: memcached steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -749,14 +750,14 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 microgateway-core: runs-on: ubuntu-latest env: PLUGINS: microgateway-core steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -766,14 +767,14 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 mocha: runs-on: ubuntu-latest env: PLUGINS: mocha steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -781,7 +782,7 @@ jobs: - run: yarn test:plugins:ci - uses: ./.github/actions/node/latest - run: yarn test:plugins:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 - if: always() uses: ./.github/actions/testagent/logs @@ -790,7 +791,7 @@ jobs: env: PLUGINS: moleculer steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -800,7 +801,7 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 mongodb-core: runs-on: ubuntu-latest @@ -813,7 +814,7 @@ jobs: PLUGINS: mongodb-core|express-mongo-sanitize SERVICES: mongo steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -823,7 +824,7 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 mongoose: runs-on: ubuntu-latest @@ -836,7 +837,7 @@ jobs: PLUGINS: mongoose SERVICES: mongo steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -846,7 +847,7 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 mysql: runs-on: ubuntu-latest @@ -862,7 +863,7 @@ jobs: PLUGINS: mysql|mysql2|mariadb # TODO: move mysql2 to its own job SERVICES: mysql steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -872,14 +873,14 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 net: runs-on: ubuntu-latest env: PLUGINS: net steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -893,7 +894,7 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 # TODO: fix performance issues and test more Node versions next: @@ -909,7 +910,7 @@ jobs: PLUGINS: next RANGE: ${{ matrix.range }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - uses: actions/setup-node@v3 @@ -919,14 +920,14 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 openai: runs-on: ubuntu-latest env: PLUGINS: openai steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -936,7 +937,7 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 opensearch: runs-on: ubuntu-latest @@ -952,7 +953,7 @@ jobs: PLUGINS: opensearch SERVICES: opensearch steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -962,7 +963,7 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 # TODO: Install the Oracle client on the host and test Node >=16. # TODO: Figure out why nyc stopped working with EACCESS errors. @@ -991,19 +992,20 @@ jobs: SERVICES: oracledb DD_TEST_AGENT_URL: http://testagent:9126 steps: - - uses: actions/checkout@v2 + # Needs to remain on v3 for now due to GLIBC version + - uses: actions/checkout@v3 - uses: ./.github/actions/node/setup - run: yarn install --ignore-engines - run: yarn services - run: yarn test:plugins - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 paperplane: runs-on: ubuntu-latest env: PLUGINS: paperplane steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -1011,7 +1013,7 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 # TODO: re-enable upstream tests if it ever stops being flaky pino: @@ -1019,7 +1021,7 @@ jobs: env: PLUGINS: pino steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -1030,7 +1032,7 @@ jobs: # - run: yarn test:plugins:upstream - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 postgres: runs-on: ubuntu-latest @@ -1046,7 +1048,7 @@ jobs: PLUGINS: pg SERVICES: postgres steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -1056,14 +1058,14 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 promise: runs-on: ubuntu-latest env: PLUGINS: promise steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -1075,14 +1077,14 @@ jobs: - run: yarn test:plugins:upstream - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 promise-js: runs-on: ubuntu-latest env: PLUGINS: promise-js steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -1092,14 +1094,14 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 q: runs-on: ubuntu-latest env: PLUGINS: q steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -1109,7 +1111,7 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 redis: runs-on: ubuntu-latest @@ -1122,7 +1124,7 @@ jobs: PLUGINS: redis|ioredis # TODO: move ioredis to its own job SERVICES: redis steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -1132,14 +1134,14 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 restify: runs-on: ubuntu-latest env: PLUGINS: restify steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -1149,14 +1151,14 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 router: runs-on: ubuntu-latest env: PLUGINS: router steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -1166,14 +1168,14 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 sharedb: runs-on: ubuntu-latest env: PLUGINS: sharedb steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -1181,7 +1183,7 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 tedious: runs-on: ubuntu-latest @@ -1198,7 +1200,7 @@ jobs: PLUGINS: tedious SERVICES: mssql steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -1207,14 +1209,14 @@ jobs: - run: yarn test:plugins:upstream - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 when: runs-on: ubuntu-latest env: PLUGINS: when steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -1224,14 +1226,14 @@ jobs: - run: yarn test:plugins:ci - if: always() uses: ./.github/actions/testagent/logs - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 winston: runs-on: ubuntu-latest env: PLUGINS: winston steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/setup - run: yarn install @@ -1240,4 +1242,4 @@ jobs: - uses: ./.github/actions/node/latest - run: yarn test:plugins:ci - if: always() - uses: ./.github/actions/testagent/logs \ No newline at end of file + uses: ./.github/actions/testagent/logs diff --git a/.github/workflows/profiling.yml b/.github/workflows/profiling.yml index 05e9696cc48..90731d062cf 100644 --- a/.github/workflows/profiling.yml +++ b/.github/workflows/profiling.yml @@ -15,16 +15,16 @@ jobs: macos: runs-on: macos-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - run: yarn install - run: yarn test:profiler:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 ubuntu: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - run: yarn install - uses: ./.github/actions/node/16 @@ -35,13 +35,13 @@ jobs: - run: yarn test:profiler:ci - uses: ./.github/actions/node/latest - run: yarn test:profiler:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 windows: runs-on: windows-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - run: yarn install - run: yarn test:profiler:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 diff --git a/.github/workflows/project.yml b/.github/workflows/project.yml index 05079d33112..34793f3e111 100644 --- a/.github/workflows/project.yml +++ b/.github/workflows/project.yml @@ -21,7 +21,7 @@ jobs: version: [16, 18, latest] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-node@v3 with: node-version: ${{ matrix.version }} @@ -41,7 +41,7 @@ jobs: DD_CIVISIBILITY_AGENTLESS_ENABLED: 1 DD_API_KEY: ${{ secrets.DD_API_KEY_CI_APP }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-node@v3 with: node-version: ${{ matrix.version }} @@ -62,7 +62,7 @@ jobs: DD_CIVISIBILITY_AGENTLESS_ENABLED: 1 DD_API_KEY: ${{ secrets.DD_API_KEY_CI_APP }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - run: yarn install - uses: actions/setup-node@v3 @@ -76,7 +76,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - run: yarn install - run: yarn lint @@ -84,7 +84,7 @@ jobs: typescript: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - run: yarn install - run: yarn type:test diff --git a/.github/workflows/release-3.yml b/.github/workflows/release-3.yml index ec25371051a..e8791ff645a 100644 --- a/.github/workflows/release-3.yml +++ b/.github/workflows/release-3.yml @@ -19,7 +19,7 @@ jobs: env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: actions/setup-node@v3 with: registry-url: 'https://registry.npmjs.org' @@ -27,7 +27,7 @@ jobs: - id: pkg run: | content=`cat ./package.json | tr '\n' ' '` - echo "::set-output name=json::$content" + echo "json=$content" >> $GITHUB_OUTPUT - run: | git tag v${{ fromJson(steps.pkg.outputs.json).version }} git push origin v${{ fromJson(steps.pkg.outputs.json).version }} @@ -36,7 +36,7 @@ jobs: runs-on: ubuntu-latest needs: ['publish'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-node@v3 - name: Log in to the Container registry uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b @@ -47,7 +47,7 @@ jobs: - id: pkg run: | content=`cat ./package.json | tr '\n' ' '` - echo "::set-output name=json::$content" + echo "json=$content" >> $GITHUB_OUTPUT - name: npm pack for injection image run: | npm pack dd-trace@${{ fromJson(steps.pkg.outputs.json).version }} diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index 936c0ee0737..fc00326a27f 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -15,7 +15,7 @@ jobs: env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-node@v3 with: registry-url: 'https://registry.npmjs.org' @@ -23,7 +23,7 @@ jobs: - id: pkg run: | content=`cat ./package.json | tr '\n' ' '` - echo "::set-output name=json::$content" + echo "json=$content" >> $GITHUB_OUTPUT - run: npm version --no-git-tag-version ${{ fromJson(steps.pkg.outputs.json).version }}-$(git rev-parse --short HEAD)+${{ github.run_id }}.${{ github.run_attempt }} - run: npm publish --tag dev --provenance - run: | @@ -35,7 +35,7 @@ jobs: runs-on: ubuntu-latest needs: ['dev_release'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-node@v3 - name: Log in to the Container registry uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b diff --git a/.github/workflows/release-latest.yml b/.github/workflows/release-latest.yml index a45ed3c87a7..b01be5a4f56 100644 --- a/.github/workflows/release-latest.yml +++ b/.github/workflows/release-latest.yml @@ -21,7 +21,7 @@ jobs: outputs: pkgjson: ${{ steps.pkg.outputs.json }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: actions/setup-node@v3 with: registry-url: 'https://registry.npmjs.org' @@ -29,7 +29,7 @@ jobs: - id: pkg run: | content=`cat ./package.json | tr '\n' ' '` - echo "::set-output name=json::$content" + echo "json=$content" >> $GITHUB_OUTPUT - run: | git tag v${{ fromJson(steps.pkg.outputs.json).version }} git push origin v${{ fromJson(steps.pkg.outputs.json).version }} @@ -38,7 +38,7 @@ jobs: runs-on: ubuntu-latest needs: ['publish'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-node@v3 - name: Log in to the Container registry uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b @@ -49,7 +49,7 @@ jobs: - id: pkg run: | content=`cat ./package.json | tr '\n' ' '` - echo "::set-output name=json::$content" + echo "json=$content" >> $GITHUB_OUTPUT - name: npm pack for injection image run: | npm pack dd-trace@${{ fromJson(steps.pkg.outputs.json).version }} @@ -61,12 +61,12 @@ jobs: runs-on: ubuntu-latest needs: ['publish'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-node@v3 - id: pkg run: | content=`cat ./package.json | tr '\n' ' '` - echo "::set-output name=json::$content" + echo "json=$content" >> $GITHUB_OUTPUT - run: yarn - name: Build working-directory: docs @@ -74,7 +74,7 @@ jobs: yarn yarn build mv out /tmp/out - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: gh-pages - name: Deploy diff --git a/.github/workflows/release-proposal.yml b/.github/workflows/release-proposal.yml index 4935f78c232..5faf193d3ef 100644 --- a/.github/workflows/release-proposal.yml +++ b/.github/workflows/release-proposal.yml @@ -8,7 +8,7 @@ jobs: check_labels: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 0 - uses: actions/setup-node@v3 diff --git a/.github/workflows/serverless-integration-test.yml b/.github/workflows/serverless-integration-test.yml index 1687b18fc22..be3eeede960 100644 --- a/.github/workflows/serverless-integration-test.yml +++ b/.github/workflows/serverless-integration-test.yml @@ -16,7 +16,7 @@ jobs: version: [16, latest] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - run: yarn install - uses: actions/setup-node@v3 diff --git a/.github/workflows/serverless-performance.yml b/.github/workflows/serverless-performance.yml index 47c330ddc4f..a23b18a9bf2 100644 --- a/.github/workflows/serverless-performance.yml +++ b/.github/workflows/serverless-performance.yml @@ -16,9 +16,9 @@ jobs: aws-runtime-name: "nodejs18.x" steps: - name: Checkout dd-trace-js - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Checkout datadog-lambda-js - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: DataDog/datadog-lambda-js path: datadog-lambda-js diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index 30503e8452b..8eca425b086 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -26,12 +26,12 @@ jobs: steps: - name: Checkout system tests - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: 'DataDog/system-tests' - name: Checkout dd-trace-js - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: 'binaries/dd-trace-js' @@ -66,14 +66,14 @@ jobs: NODEJS_DDTRACE_MODULE: datadog/dd-trace-js#${{ github.sha }} steps: - name: Checkout system tests - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: 'DataDog/system-tests' - uses: actions/setup-python@v4 with: python-version: '3.9' - name: Checkout dd-trace-js - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: 'binaries/dd-trace-js' - name: Build diff --git a/.github/workflows/test-k8s-lib-injection.yaml b/.github/workflows/test-k8s-lib-injection.yaml index d489708c06a..d0e971e420b 100644 --- a/.github/workflows/test-k8s-lib-injection.yaml +++ b/.github/workflows/test-k8s-lib-injection.yaml @@ -26,7 +26,7 @@ jobs: id: set_names run: | echo "Docker image tag: $(echo ${GITHUB_HEAD_REF-${GITHUB_REF#refs/heads/}} | tr / -)" - echo "::set-output name=image_name::$(echo ${GITHUB_HEAD_REF-${GITHUB_REF#refs/heads/}} | tr / -)" + echo "image_name=$(echo ${GITHUB_HEAD_REF-${GITHUB_REF#refs/heads/}} | tr / -)" >> $GITHUB_OUTPUT - name: Npm pack for injection image run: | diff --git a/.github/workflows/tracing.yml b/.github/workflows/tracing.yml index 1b580a24aa3..3062df0166a 100644 --- a/.github/workflows/tracing.yml +++ b/.github/workflows/tracing.yml @@ -15,16 +15,16 @@ jobs: macos: runs-on: macos-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - run: yarn install - run: yarn test:trace:core:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 ubuntu: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - run: yarn install - uses: ./.github/actions/node/16 @@ -35,13 +35,13 @@ jobs: - run: yarn test:trace:core:ci - uses: ./.github/actions/node/latest - run: yarn test:trace:core:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 windows: runs-on: windows-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: ./.github/actions/node/setup - run: yarn install - run: yarn test:trace:core:ci - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 From e84b27d4faf354febb79dd8b6253d064619a66e3 Mon Sep 17 00:00:00 2001 From: Bowen Brooks <39347269+bojbrook@users.noreply.github.com> Date: Tue, 2 Jan 2024 07:06:38 -0500 Subject: [PATCH 13/21] Adding Pino Integration-test (#2002) --- integration-tests/pino.spec.js | 72 +++++++++++++++++++++++++++++++++ integration-tests/pino/index.js | 32 +++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 integration-tests/pino.spec.js create mode 100644 integration-tests/pino/index.js diff --git a/integration-tests/pino.spec.js b/integration-tests/pino.spec.js new file mode 100644 index 00000000000..a28bf72a08b --- /dev/null +++ b/integration-tests/pino.spec.js @@ -0,0 +1,72 @@ +/* eslint-disable comma-dangle */ +'use strict' + +const { FakeAgent, spawnProc, createSandbox, curl } = require('./helpers') +const path = require('path') +const { assert } = require('chai') +const { once } = require('events') + +describe('pino test', () => { + let agent + let proc + let sandbox + let cwd + let startupTestFile + + before(async () => { + sandbox = await createSandbox(['pino']) + cwd = sandbox.folder + startupTestFile = path.join(cwd, 'pino/index.js') + }) + + after(async () => { + await sandbox.remove() + }) + + context('Log injection', () => { + beforeEach(async () => { + agent = await new FakeAgent().start() + }) + + afterEach(async () => { + proc.kill() + await agent.stop() + }) + + it('Log injection enabled', async () => { + proc = await spawnProc(startupTestFile, { + cwd, + env: { + AGENT_PORT: agent.port, + lOG_INJECTION: true, + }, + stdio: 'pipe', + }) + const [data] = await Promise.all([once(proc.stdout, 'data'), curl(proc)]) + const stdoutData = JSON.parse(data.toString()) + assert.containsAllKeys(stdoutData, ['dd']) + assert.containsAllKeys(stdoutData.dd, ['trace_id', 'span_id']) + assert.strictEqual( + stdoutData['dd']['trace_id'], + stdoutData['custom']['trace_id'] + ) + assert.strictEqual( + stdoutData['dd']['span_id'], + stdoutData['custom']['span_id'] + ) + }) + + it('Log injection disabled', async () => { + proc = await spawnProc(startupTestFile, { + cwd, + env: { + AGENT_PORT: agent.port, + }, + stdio: 'pipe', + }) + const [data] = await Promise.all([once(proc.stdout, 'data'), curl(proc)]) + const stdoutData = JSON.parse(data.toString()) + assert.doesNotHaveAnyKeys(stdoutData, ['dd']) + }) + }) +}) diff --git a/integration-tests/pino/index.js b/integration-tests/pino/index.js new file mode 100644 index 00000000000..40e35388fac --- /dev/null +++ b/integration-tests/pino/index.js @@ -0,0 +1,32 @@ +'use strict' + +const options = {} + +if (process.env.AGENT_PORT) { + options.port = process.env.AGENT_PORT +} + +if (process.env.lOG_INJECTION) { + options.logInjection = process.env.lOG_INJECTION +} + +const tracer = require('dd-trace').init(options) + +const http = require('http') +const logger = require('pino')() + +const server = http + .createServer((req, res) => { + const span = tracer.scope().active() + const contextTraceId = span.context().toTraceId() + const contextSpanId = span.context().toSpanId() + logger.info( + { custom: { trace_id: contextTraceId, span_id: contextSpanId } }, + 'Creating server' + ) + res.end('hello, world\n') + }) + .listen(0, () => { + const port = server.address().port + process.send({ port }) + }) From 53143f354b4832c36e2e3bf73f326a40cdf7c21d Mon Sep 17 00:00:00 2001 From: Nicolas Savoire Date: Tue, 2 Jan 2024 13:13:34 +0100 Subject: [PATCH 14/21] Add `process_id` tag to profiles (#3911) Add a `process_id` that contains process pid to profiles. --- packages/dd-trace/src/profiling/exporters/agent.js | 1 + packages/dd-trace/test/profiling/exporters/agent.spec.js | 2 ++ 2 files changed, 3 insertions(+) diff --git a/packages/dd-trace/src/profiling/exporters/agent.js b/packages/dd-trace/src/profiling/exporters/agent.js index 712d03f1406..517b774890e 100644 --- a/packages/dd-trace/src/profiling/exporters/agent.js +++ b/packages/dd-trace/src/profiling/exporters/agent.js @@ -75,6 +75,7 @@ class AgentExporter { ['tags[]', 'language:javascript'], ['tags[]', 'runtime:nodejs'], ['tags[]', `runtime_version:${process.version}`], + ['tags[]', `process_id:${process.pid}`], ['tags[]', `profiler_version:${version}`], ['tags[]', 'format:pprof'], ...Object.entries(tags).map(([key, value]) => ['tags[]', `${key}:${value}`]) diff --git a/packages/dd-trace/test/profiling/exporters/agent.spec.js b/packages/dd-trace/test/profiling/exporters/agent.spec.js index 92e7f097539..c1a1038d95c 100644 --- a/packages/dd-trace/test/profiling/exporters/agent.spec.js +++ b/packages/dd-trace/test/profiling/exporters/agent.spec.js @@ -75,6 +75,7 @@ describe('exporters/agent', function () { 'language:javascript', 'runtime:nodejs', `runtime_version:${process.version}`, + `process_id:${process.pid}`, `profiler_version:${version}`, 'format:pprof', 'runtime-id:a1b2c3d4-a1b2-a1b2-a1b2-a1b2c3d4e5f6' @@ -359,6 +360,7 @@ describe('exporters/agent', function () { 'language:javascript', 'runtime:nodejs', `runtime_version:${process.version}`, + `process_id:${process.pid}`, `profiler_version:${version}`, 'format:pprof', 'foo:bar' From 1a94f5c35d990b62ba0dceff14f45ad4d9fc30f8 Mon Sep 17 00:00:00 2001 From: Roberto Montero <108007532+robertomonteromiguel@users.noreply.github.com> Date: Wed, 3 Jan 2024 10:11:57 +0100 Subject: [PATCH 15/21] Parametric tests: Remove obsolete CI code (#3922) --- .github/workflows/system-tests.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index 8eca425b086..43155c47968 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -62,8 +62,6 @@ jobs: runs-on: ubuntu-latest env: TEST_LIBRARY: nodejs - #keep this line until system-tests -> robertomonteromiguel/parametric_use_load_binary_nodejs merged - NODEJS_DDTRACE_MODULE: datadog/dd-trace-js#${{ github.sha }} steps: - name: Checkout system tests uses: actions/checkout@v4 From b93d6fa94ffb8cffe3f3353dab5fbae46c8117ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Wed, 3 Jan 2024 12:15:12 +0100 Subject: [PATCH 16/21] speed up test by stubbing (#3923) --- .../dd-trace/test/plugins/util/test-environment.spec.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/dd-trace/test/plugins/util/test-environment.spec.js b/packages/dd-trace/test/plugins/util/test-environment.spec.js index 181c7f6fe52..547e2521b58 100644 --- a/packages/dd-trace/test/plugins/util/test-environment.spec.js +++ b/packages/dd-trace/test/plugins/util/test-environment.spec.js @@ -6,14 +6,14 @@ const fs = require('fs') const path = require('path') const proxyquire = require('proxyquire') -const sanitizedExecStub = sinon.stub().returns('') +const execFileSyncStub = sinon.stub().returns('') const { getCIMetadata } = require('../../../src/plugins/util/ci') const { CI_ENV_VARS, CI_NODE_LABELS } = require('../../../src/plugins/util/tags') const { getGitMetadata } = proxyquire('../../../src/plugins/util/git', { - './exec': { - 'sanitizedExec': sanitizedExecStub + 'child_process': { + 'execFileSync': execFileSyncStub } }) const { getTestEnvironmentMetadata } = proxyquire('../../../src/plugins/util/test', { From 8554e5be5638196759fa9c268c50709b8baff41f Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Thu, 4 Jan 2024 02:06:48 +0800 Subject: [PATCH 17/21] Implement extended sampling (#3904) --- packages/dd-trace/src/priority_sampler.js | 68 +- packages/dd-trace/src/sampling_rule.js | 130 ++++ packages/dd-trace/src/span_sampler.js | 70 +- .../dd-trace/test/priority_sampler.spec.js | 66 +- packages/dd-trace/test/sampling_rule.spec.js | 409 ++++++++++ packages/dd-trace/test/span_sampler.spec.js | 730 ++++++------------ 6 files changed, 804 insertions(+), 669 deletions(-) create mode 100644 packages/dd-trace/src/sampling_rule.js create mode 100644 packages/dd-trace/test/sampling_rule.spec.js diff --git a/packages/dd-trace/src/priority_sampler.js b/packages/dd-trace/src/priority_sampler.js index 40ca1833cca..2c7b22b2854 100644 --- a/packages/dd-trace/src/priority_sampler.js +++ b/packages/dd-trace/src/priority_sampler.js @@ -1,9 +1,8 @@ 'use strict' -const RateLimiter = require('./rate_limiter') const Sampler = require('./sampler') -const ext = require('../../../ext') const { setSamplingRules } = require('./startup-log') +const SamplingRule = require('./sampling_rule') const { SAMPLING_MECHANISM_DEFAULT, @@ -16,14 +15,21 @@ const { DECISION_MAKER_KEY } = require('./constants') -const SERVICE_NAME = ext.tags.SERVICE_NAME -const SAMPLING_PRIORITY = ext.tags.SAMPLING_PRIORITY -const MANUAL_KEEP = ext.tags.MANUAL_KEEP -const MANUAL_DROP = ext.tags.MANUAL_DROP -const USER_REJECT = ext.priority.USER_REJECT -const AUTO_REJECT = ext.priority.AUTO_REJECT -const AUTO_KEEP = ext.priority.AUTO_KEEP -const USER_KEEP = ext.priority.USER_KEEP +const { + tags: { + MANUAL_KEEP, + MANUAL_DROP, + SAMPLING_PRIORITY, + SERVICE_NAME + }, + priority: { + AUTO_REJECT, + AUTO_KEEP, + USER_REJECT, + USER_KEEP + } +} = require('../../../ext') + const DEFAULT_KEY = 'service:,env:' const defaultSampler = new Sampler(AUTO_KEEP) @@ -36,8 +42,7 @@ class PrioritySampler { configure (env, { sampleRate, rateLimit = 100, rules = [] } = {}) { this._env = env - this._rules = this._normalizeRules(rules, sampleRate) - this._limiter = new RateLimiter(rateLimit) + this._rules = this._normalizeRules(rules, sampleRate, rateLimit) setSamplingRules(this._rules) } @@ -104,7 +109,7 @@ class PrioritySampler { _getPriorityFromAuto (span) { const context = this._getContext(span) - const rule = this._findRule(context) + const rule = this._findRule(span) return rule ? this._getPriorityByRule(context, rule) @@ -131,15 +136,14 @@ class PrioritySampler { context._trace[SAMPLING_RULE_DECISION] = rule.sampleRate context._sampling.mechanism = SAMPLING_MECHANISM_RULE - return rule.sampler.isSampled(context) && this._isSampledByRateLimit(context) ? USER_KEEP : USER_REJECT - } + const sampled = rule.sample() + const priority = sampled ? USER_KEEP : USER_REJECT - _isSampledByRateLimit (context) { - const allowed = this._limiter.isAllowed() - - context._trace[SAMPLING_LIMIT_DECISION] = this._limiter.effectiveRate() + if (sampled) { + context._trace[SAMPLING_LIMIT_DECISION] = rule.effectiveRate + } - return allowed + return priority } _getPriorityByAgent (context) { @@ -172,33 +176,21 @@ class PrioritySampler { } } - _normalizeRules (rules, sampleRate) { + _normalizeRules (rules, sampleRate, rateLimit) { rules = [].concat(rules || []) return rules - .concat({ sampleRate }) + .concat({ sampleRate, maxPerSecond: rateLimit }) .map(rule => ({ ...rule, sampleRate: parseFloat(rule.sampleRate) })) .filter(rule => !isNaN(rule.sampleRate)) - .map(rule => ({ ...rule, sampler: new Sampler(rule.sampleRate) })) + .map(SamplingRule.from) } - _findRule (context) { - for (let i = 0, l = this._rules.length; i < l; i++) { - if (this._matchRule(context, this._rules[i])) return this._rules[i] + _findRule (span) { + for (const rule of this._rules) { + if (rule.match(span)) return rule } } - - _matchRule (context, rule) { - const name = context._name - const service = context._tags['service.name'] - - if (rule.name instanceof RegExp && !rule.name.test(name)) return false - if (typeof rule.name === 'string' && rule.name !== name) return false - if (rule.service instanceof RegExp && !rule.service.test(service)) return false - if (typeof rule.service === 'string' && rule.service !== service) return false - - return true - } } function hasOwn (object, prop) { diff --git a/packages/dd-trace/src/sampling_rule.js b/packages/dd-trace/src/sampling_rule.js new file mode 100644 index 00000000000..b3181aef641 --- /dev/null +++ b/packages/dd-trace/src/sampling_rule.js @@ -0,0 +1,130 @@ +'use strict' + +const { globMatch } = require('../src/util') +const RateLimiter = require('./rate_limiter') +const Sampler = require('./sampler') + +class AlwaysMatcher { + match () { + return true + } +} + +class GlobMatcher { + constructor (pattern, locator) { + this.pattern = pattern + this.locator = locator + } + + match (span) { + const subject = this.locator(span) + if (!subject) return false + return globMatch(this.pattern, subject) + } +} + +class RegExpMatcher { + constructor (pattern, locator) { + this.pattern = pattern + this.locator = locator + } + + match (span) { + const subject = this.locator(span) + if (!subject) return false + return this.pattern.test(subject) + } +} + +function matcher (pattern, locator) { + if (pattern instanceof RegExp) { + return new RegExpMatcher(pattern, locator) + } + + if (typeof pattern === 'string' && pattern !== '*') { + return new GlobMatcher(pattern, locator) + } + + return new AlwaysMatcher() +} + +function makeTagLocator (tag) { + return (span) => span.context()._tags[tag] +} + +function nameLocator (span) { + return span.context()._name +} + +function serviceLocator (span) { + const { _tags: tags } = span.context() + return tags.service || + tags['service.name'] || + span.tracer()._service +} + +class SamplingRule { + constructor ({ name, service, resource, tags, sampleRate = 1.0, maxPerSecond } = {}) { + this.matchers = [] + + if (name) { + this.matchers.push(matcher(name, nameLocator)) + } + if (service) { + this.matchers.push(matcher(service, serviceLocator)) + } + if (resource) { + this.matchers.push(matcher(resource, makeTagLocator('resource.name'))) + } + for (const [key, value] of Object.entries(tags || {})) { + this.matchers.push(matcher(value, makeTagLocator(key))) + } + + this._sampler = new Sampler(sampleRate) + this._limiter = undefined + + if (Number.isFinite(maxPerSecond)) { + this._limiter = new RateLimiter(maxPerSecond) + } + } + + static from (config) { + return new SamplingRule(config) + } + + get sampleRate () { + return this._sampler.rate() + } + + get effectiveRate () { + return this._limiter && this._limiter.effectiveRate() + } + + get maxPerSecond () { + return this._limiter && this._limiter._rateLimit + } + + match (span) { + for (const matcher of this.matchers) { + if (!matcher.match(span)) { + return false + } + } + + return true + } + + sample () { + if (!this._sampler.isSampled()) { + return false + } + + if (this._limiter) { + return this._limiter.isAllowed() + } + + return true + } +} + +module.exports = SamplingRule diff --git a/packages/dd-trace/src/span_sampler.js b/packages/dd-trace/src/span_sampler.js index d43fba63cc8..0a8900b5db5 100644 --- a/packages/dd-trace/src/span_sampler.js +++ b/packages/dd-trace/src/span_sampler.js @@ -1,67 +1,16 @@ 'use strict' -const { globMatch } = require('../src/util') -const { USER_KEEP, AUTO_KEEP } = require('../../../ext').priority -const RateLimiter = require('./rate_limiter') -const Sampler = require('./sampler') - -class SpanSamplingRule { - constructor ({ service, name, sampleRate = 1.0, maxPerSecond } = {}) { - this.service = service - this.name = name - - this._sampler = new Sampler(sampleRate) - this._limiter = undefined - - if (Number.isFinite(maxPerSecond)) { - this._limiter = new RateLimiter(maxPerSecond) - } - } - - get sampleRate () { - return this._sampler.rate() - } - - get maxPerSecond () { - return this._limiter && this._limiter._rateLimit - } - - static from (config) { - return new SpanSamplingRule(config) - } - - match (service, name) { - if (this.service && !globMatch(this.service, service)) { - return false - } - - if (this.name && !globMatch(this.name, name)) { - return false - } - - return true - } - - sample () { - if (!this._sampler.isSampled()) { - return false - } - if (this._limiter) { - return this._limiter.isAllowed() - } - - return true - } -} +const { USER_KEEP, AUTO_KEEP } = require('../../../ext').priority +const SamplingRule = require('./sampling_rule') class SpanSampler { constructor ({ spanSamplingRules = [] } = {}) { - this._rules = spanSamplingRules.map(SpanSamplingRule.from) + this._rules = spanSamplingRules.map(SamplingRule.from) } - findRule (service, name) { + findRule (context) { for (const rule of this._rules) { - if (rule.match(service, name)) { + if (rule.match(context)) { return rule } } @@ -73,14 +22,7 @@ class SpanSampler { const { started } = spanContext._trace for (const span of started) { - const context = span.context() - const tags = context._tags || {} - const name = context._name - const service = tags.service || - tags['service.name'] || - span.tracer()._service - - const rule = this.findRule(service, name) + const rule = this.findRule(span) if (rule && rule.sample()) { span.context()._spanSampling = { sampleRate: rule.sampleRate, diff --git a/packages/dd-trace/test/priority_sampler.spec.js b/packages/dd-trace/test/priority_sampler.spec.js index 00753c79a44..7bb2b246663 100644 --- a/packages/dd-trace/test/priority_sampler.spec.js +++ b/packages/dd-trace/test/priority_sampler.spec.js @@ -24,6 +24,7 @@ const USER_KEEP = ext.priority.USER_KEEP describe('PrioritySampler', () => { let PrioritySampler let prioritySampler + let SamplingRule let Sampler let sampler let context @@ -32,7 +33,8 @@ describe('PrioritySampler', () => { beforeEach(() => { context = { _tags: { - 'service.name': 'test' + 'service.name': 'test', + 'resource.name': 'resource' }, _sampling: {}, _trace: { @@ -65,10 +67,15 @@ describe('PrioritySampler', () => { }) Sampler.withArgs(0.5).returns(sampler) - PrioritySampler = proxyquire('../src/priority_sampler', { + SamplingRule = proxyquire('../src/sampling_rule', { './sampler': Sampler }) + PrioritySampler = proxyquire('../src/priority_sampler', { + './sampler': Sampler, + './sampling_rule': SamplingRule + }) + prioritySampler = new PrioritySampler('test') }) @@ -182,60 +189,11 @@ describe('PrioritySampler', () => { expect(context._sampling.mechanism).to.equal(SAMPLING_MECHANISM_RULE) }) - it('should support a sample rate from a rule on service as string', () => { - context._tags['service.name'] = 'test' - - prioritySampler = new PrioritySampler('test', { - rules: [ - { sampleRate: 0, service: 'foo' }, - { sampleRate: 1, service: 'test' } - ] - }) - prioritySampler.sample(context) - - expect(context._sampling).to.have.property('priority', USER_KEEP) - expect(context._sampling.mechanism).to.equal(SAMPLING_MECHANISM_RULE) - }) - - it('should support a sample rate from a rule on service as regex', () => { - context._tags['service.name'] = 'test' - - prioritySampler = new PrioritySampler('test', { - rules: [ - { sampleRate: 0, service: /fo/ }, - { sampleRate: 1, service: /tes/ } - ] - }) - prioritySampler.sample(context) - - expect(context._sampling).to.have.property('priority', USER_KEEP) - expect(context._sampling.mechanism).to.equal(SAMPLING_MECHANISM_RULE) - }) - - it('should support a sample rate from a rule on name as string', () => { - context._name = 'foo' - context._tags['service.name'] = 'test' - - prioritySampler = new PrioritySampler('test', { - rules: [ - { sampleRate: 0, name: 'bar' }, - { sampleRate: 1, name: 'foo' } - ] - }) - prioritySampler.sample(context) - - expect(context._sampling).to.have.property('priority', USER_KEEP) - expect(context._sampling.mechanism).to.equal(SAMPLING_MECHANISM_RULE) - }) - - it('should support a sample rate from a rule on name as regex', () => { - context._name = 'foo' - context._tags['service.name'] = 'test' - + it('should support a rule-based sampling', () => { prioritySampler = new PrioritySampler('test', { rules: [ - { sampleRate: 0, name: /ba/ }, - { sampleRate: 1, name: /fo/ } + { sampleRate: 0, service: 'foo', resource: /res.*/ }, + { sampleRate: 1, service: 'test', resource: /res.*/ } ] }) prioritySampler.sample(context) diff --git a/packages/dd-trace/test/sampling_rule.spec.js b/packages/dd-trace/test/sampling_rule.spec.js new file mode 100644 index 00000000000..27b2a152c90 --- /dev/null +++ b/packages/dd-trace/test/sampling_rule.spec.js @@ -0,0 +1,409 @@ +'use strict' + +require('./setup/tap') + +const { expect } = require('chai') +const id = require('../src/id') + +function createDummySpans () { + const operations = [ + 'operation', + 'sub_operation', + 'second_operation', + 'sub_second_operation_1', + 'sub_second_operation_2', + 'sub_sub_second_operation_2', + 'custom_service_span_1', + 'custom_service_span_2', + 'renamed_operation', + 'tagged_operation', + 'resource_named_operation' + ] + + const ids = [ + id('0234567812345671'), + id('0234567812345672'), + id('0234567812345673'), + id('0234567812345674'), + id('0234567812345675'), + id('0234567812345676'), + id('0234567812345677'), + id('0234567812345678'), + id('0234567812345679'), + id('0234567812345680'), + id('0234567812345681') + ] + + const spans = [] + const spanContexts = [] + + for (let idx = 0; idx < operations.length; idx++) { + const operation = operations[idx] + const id = ids[idx] + const spanContext = { + _spanId: id, + _sampling: {}, + _trace: { + started: [] + }, + _name: operation, + _tags: {} + } + + // Give first span a custom service name + if ([6, 7].includes(idx)) { + spanContext._tags['service.name'] = 'span-service' + } + + if (idx === 8) { + spanContext._name = 'renamed' + } + + if (idx === 9) { + spanContext._tags['tagged'] = 'yup' + spanContext._tags['and'] = 'this' + spanContext._tags['not'] = 'this' + } + + if (idx === 10) { + spanContext._tags['resource.name'] = 'named' + } + + const span = { + context: sinon.stub().returns(spanContext), + tracer: sinon.stub().returns({ + _service: 'test' + }), + _name: operation + } + + spanContexts.push(spanContext) + spans.push(span) + } + + return { spans, spanContexts } +} + +describe('sampling rule', () => { + let spans + let spanContexts + let SamplingRule + let rule + + beforeEach(() => { + const info = createDummySpans() + spans = info.spans + spanContexts = info.spanContexts + + spanContexts[0]._trace.started.push(...spans) + + SamplingRule = require('../src/sampling_rule') + }) + + describe('match', () => { + it('should match with exact strings', () => { + rule = new SamplingRule({ + service: 'test', + name: 'operation' + }) + + expect(rule.match(spans[0])).to.equal(true) + expect(rule.match(spans[1])).to.equal(false) + expect(rule.match(spans[2])).to.equal(false) + expect(rule.match(spans[3])).to.equal(false) + expect(rule.match(spans[4])).to.equal(false) + expect(rule.match(spans[5])).to.equal(false) + expect(rule.match(spans[6])).to.equal(false) + expect(rule.match(spans[7])).to.equal(false) + expect(rule.match(spans[8])).to.equal(false) + expect(rule.match(spans[9])).to.equal(false) + expect(rule.match(spans[10])).to.equal(false) + }) + + it('should match with regexp', () => { + rule = new SamplingRule({ + service: /test/, + name: /op.*/ + }) + + expect(rule.match(spans[0])).to.equal(true) + expect(rule.match(spans[1])).to.equal(true) + expect(rule.match(spans[2])).to.equal(true) + expect(rule.match(spans[3])).to.equal(true) + expect(rule.match(spans[4])).to.equal(true) + expect(rule.match(spans[5])).to.equal(true) + expect(rule.match(spans[6])).to.equal(false) + expect(rule.match(spans[7])).to.equal(false) + expect(rule.match(spans[8])).to.equal(false) + expect(rule.match(spans[9])).to.equal(true) + expect(rule.match(spans[10])).to.equal(true) + }) + + it('should match with postfix glob', () => { + rule = new SamplingRule({ + service: 'test', + name: 'op*' + }) + + expect(rule.match(spans[0])).to.equal(true) + expect(rule.match(spans[1])).to.equal(false) + expect(rule.match(spans[2])).to.equal(false) + expect(rule.match(spans[3])).to.equal(false) + expect(rule.match(spans[4])).to.equal(false) + expect(rule.match(spans[5])).to.equal(false) + expect(rule.match(spans[6])).to.equal(false) + expect(rule.match(spans[7])).to.equal(false) + expect(rule.match(spans[8])).to.equal(false) + expect(rule.match(spans[9])).to.equal(false) + expect(rule.match(spans[10])).to.equal(false) + }) + + it('should match with prefix glob', () => { + rule = new SamplingRule({ + service: 'test', + name: '*operation' + }) + + expect(rule.match(spans[0])).to.equal(true) + expect(rule.match(spans[1])).to.equal(true) + expect(rule.match(spans[2])).to.equal(true) + expect(rule.match(spans[3])).to.equal(false) + expect(rule.match(spans[4])).to.equal(false) + expect(rule.match(spans[5])).to.equal(false) + expect(rule.match(spans[6])).to.equal(false) + expect(rule.match(spans[7])).to.equal(false) + expect(rule.match(spans[8])).to.equal(false) + expect(rule.match(spans[9])).to.equal(true) + expect(rule.match(spans[10])).to.equal(true) + }) + + it('should match with single character any matcher', () => { + rule = new SamplingRule({ + service: 'test', + name: 'o?eration' + }) + + expect(rule.match(spans[0])).to.equal(true) + expect(rule.match(spans[1])).to.equal(false) + expect(rule.match(spans[2])).to.equal(false) + expect(rule.match(spans[3])).to.equal(false) + expect(rule.match(spans[4])).to.equal(false) + expect(rule.match(spans[5])).to.equal(false) + expect(rule.match(spans[6])).to.equal(false) + expect(rule.match(spans[7])).to.equal(false) + expect(rule.match(spans[8])).to.equal(false) + expect(rule.match(spans[9])).to.equal(false) + expect(rule.match(spans[10])).to.equal(false) + }) + + it('should consider missing service as match-all for service name', () => { + rule = new SamplingRule({ + name: 'sub_second_operation_*' + }) + + expect(rule.match(spans[0])).to.equal(false) + expect(rule.match(spans[1])).to.equal(false) + expect(rule.match(spans[2])).to.equal(false) + // Only 3 and 4 should match because of the name pattern + expect(rule.match(spans[3])).to.equal(true) + expect(rule.match(spans[4])).to.equal(true) + expect(rule.match(spans[5])).to.equal(false) + expect(rule.match(spans[6])).to.equal(false) + expect(rule.match(spans[7])).to.equal(false) + expect(rule.match(spans[8])).to.equal(false) + expect(rule.match(spans[9])).to.equal(false) + expect(rule.match(spans[10])).to.equal(false) + }) + + it('should consider missing name as match-all for span name', () => { + rule = new SamplingRule({ + service: 'test' + }) + + expect(rule.match(spans[0])).to.equal(true) + expect(rule.match(spans[1])).to.equal(true) + expect(rule.match(spans[2])).to.equal(true) + expect(rule.match(spans[3])).to.equal(true) + expect(rule.match(spans[4])).to.equal(true) + expect(rule.match(spans[5])).to.equal(true) + expect(rule.match(spans[8])).to.equal(true) + expect(rule.match(spans[9])).to.equal(true) + expect(rule.match(spans[10])).to.equal(true) + // Should not match because of different service name + expect(rule.match(spans[6])).to.equal(false) + expect(rule.match(spans[7])).to.equal(false) + }) + + it('should use span service name tags where present', () => { + rule = new SamplingRule({ + service: 'span-service' + }) + + expect(rule.match(spans[0])).to.equal(false) + expect(rule.match(spans[1])).to.equal(false) + expect(rule.match(spans[2])).to.equal(false) + expect(rule.match(spans[3])).to.equal(false) + expect(rule.match(spans[4])).to.equal(false) + expect(rule.match(spans[5])).to.equal(false) + expect(rule.match(spans[6])).to.equal(true) + expect(rule.match(spans[7])).to.equal(true) + expect(rule.match(spans[8])).to.equal(false) + expect(rule.match(spans[9])).to.equal(false) + expect(rule.match(spans[10])).to.equal(false) + }) + + it('should match renamed spans', () => { + rule = new SamplingRule({ + service: 'test', + name: 'renamed' + }) + + expect(rule.match(spans[0])).to.equal(false) + expect(rule.match(spans[1])).to.equal(false) + expect(rule.match(spans[2])).to.equal(false) + expect(rule.match(spans[3])).to.equal(false) + expect(rule.match(spans[4])).to.equal(false) + expect(rule.match(spans[5])).to.equal(false) + expect(rule.match(spans[6])).to.equal(false) + expect(rule.match(spans[7])).to.equal(false) + expect(rule.match(spans[8])).to.equal(true) + expect(rule.match(spans[9])).to.equal(false) + expect(rule.match(spans[10])).to.equal(false) + }) + + it('should match tag sets', () => { + rule = new SamplingRule({ + tags: { + tagged: 'yup', + and: 'this' + } + }) + + expect(rule.match(spans[0])).to.equal(false) + expect(rule.match(spans[1])).to.equal(false) + expect(rule.match(spans[2])).to.equal(false) + expect(rule.match(spans[3])).to.equal(false) + expect(rule.match(spans[4])).to.equal(false) + expect(rule.match(spans[5])).to.equal(false) + expect(rule.match(spans[6])).to.equal(false) + expect(rule.match(spans[7])).to.equal(false) + expect(rule.match(spans[8])).to.equal(false) + expect(rule.match(spans[9])).to.equal(true) + expect(rule.match(spans[10])).to.equal(false) + }) + + it('should match resource name', () => { + rule = new SamplingRule({ + resource: 'named' + }) + + expect(rule.match(spans[0])).to.equal(false) + expect(rule.match(spans[1])).to.equal(false) + expect(rule.match(spans[2])).to.equal(false) + expect(rule.match(spans[3])).to.equal(false) + expect(rule.match(spans[4])).to.equal(false) + expect(rule.match(spans[5])).to.equal(false) + expect(rule.match(spans[6])).to.equal(false) + expect(rule.match(spans[7])).to.equal(false) + expect(rule.match(spans[8])).to.equal(false) + expect(rule.match(spans[9])).to.equal(false) + expect(rule.match(spans[10])).to.equal(true) + }) + }) + + describe('sampleRate', () => { + beforeEach(() => { + sinon.stub(Math, 'random').returns(0.5) + }) + + afterEach(() => { + Math.random.restore() + }) + + it('should sample on allowed sample rate', () => { + rule = new SamplingRule({ + service: 'test', + name: 'operation', + sampleRate: 1.0, + maxPerSecond: 1 + }) + + expect(rule.sample()).to.equal(true) + }) + + it('should not sample on non-allowed sample rate', () => { + rule = new SamplingRule({ + service: 'test', + name: 'operation', + sampleRate: 0.3, + maxPerSecond: 1 + }) + + expect(rule.sample()).to.equal(false) + }) + }) + + describe('maxPerSecond', () => { + it('should not create limiter without finite maxPerSecond', () => { + rule = new SamplingRule({ + service: 'test', + name: 'operation', + sampleRate: 1.0 + }) + + expect(rule._limiter).to.equal(undefined) + expect(rule.maxPerSecond).to.equal(undefined) + }) + + it('should create limiter with finite maxPerSecond', () => { + rule = new SamplingRule({ + service: 'test', + name: 'operation', + sampleRate: 1.0, + maxPerSecond: 123 + }) + + expect(rule._limiter).to.not.equal(undefined) + expect(rule).to.have.property('maxPerSecond', 123) + }) + + it('should not sample spans past the rate limit', () => { + rule = new SamplingRule({ + service: 'test', + name: 'operation', + sampleRate: 1.0, + maxPerSecond: 1 + }) + + expect(rule.sample()).to.equal(true) + expect(rule.sample()).to.equal(false) + }) + + it('should allow unlimited rate limits', () => { + rule = new SamplingRule({ + service: 'test', + name: 'operation', + sampleRate: 1.0 + }) + + for (let i = 0; i < 1e3; i++) { + expect(rule.sample()).to.equal(true) + } + }) + + it('should sample if enough time has elapsed', () => { + rule = new SamplingRule({ + service: 'test', + name: 'operation', + sampleRate: 1.0, + maxPerSecond: 1 + }) + + const clock = sinon.useFakeTimers(new Date()) + expect(rule.sample()).to.equal(true) + expect(rule.sample()).to.equal(false) + clock.tick(1000) + expect(rule.sample()).to.equal(true) + }) + }) +}) diff --git a/packages/dd-trace/test/span_sampler.spec.js b/packages/dd-trace/test/span_sampler.spec.js index aa79047f7b5..cebc524478c 100644 --- a/packages/dd-trace/test/span_sampler.spec.js +++ b/packages/dd-trace/test/span_sampler.spec.js @@ -2,575 +2,279 @@ require('./setup/tap') -const { expect } = require('chai') const id = require('../src/id') -function createDummySpans () { - const operations = [ - 'operation', - 'sub_operation', - 'second_operation', - 'sub_second_operation_1', - 'sub_second_operation_2', - 'sub_sub_second_operation_2', - 'custom_service_span_1', - 'custom_service_span_2', - 'renamed_operation' - ] +describe('span sampler', () => { + const spies = {} + let SpanSampler + let SamplingRule - const ids = [ - id('0234567812345671'), - id('0234567812345672'), - id('0234567812345673'), - id('0234567812345674'), - id('0234567812345675'), - id('0234567812345676'), - id('0234567812345677') - ] + beforeEach(() => { + if (!SamplingRule) { + SamplingRule = require('../src/sampling_rule') + spies.match = sinon.spy(SamplingRule.prototype, 'match') + spies.sample = sinon.spy(SamplingRule.prototype, 'sample') + spies.sampleRate = sinon.spy(SamplingRule.prototype, 'sampleRate', ['get']) + spies.maxPerSecond = sinon.spy(SamplingRule.prototype, 'maxPerSecond', ['get']) + } + + SpanSampler = proxyquire('../src/span_sampler', { + './sampling_rule': SamplingRule + }) + }) - const spans = [] - const spanContexts = [] + it('should not sample anything when trace is kept', done => { + const sampler = new SpanSampler({}) - for (let idx = 0; idx < operations.length; idx++) { - const operation = operations[idx] - const id = ids[idx] const spanContext = { - _spanId: id, - _sampling: {}, + _spanId: id('1234567812345678'), + _sampling: { + priority: 2 + }, _trace: { started: [] }, - _name: operation, + _name: 'operation', _tags: {} } - - // Give first span a custom service name - if ([6, 7].includes(idx)) { - spanContext._tags['service.name'] = 'span-service' - } - - if (idx === 8) { - spanContext._name = 'renamed' - } - - const span = { + spanContext._trace.started.push({ context: sinon.stub().returns(spanContext), tracer: sinon.stub().returns({ _service: 'test' }), - _name: operation - } - - spanContexts.push(spanContext) - spans.push(span) - } - - return { spans, spanContexts } -} - -describe('span sampler', () => { - let spans - let spanContexts - let SpanSampler - let sampler - - beforeEach(() => { - const info = createDummySpans() - spans = info.spans - spanContexts = info.spanContexts - - spanContexts[0]._trace.started.push(...spans) - - SpanSampler = require('../src/span_sampler') - }) - - describe('without drop', () => { - beforeEach(() => { - spanContexts[0]._sampling.priority = 2 // user keep + _name: 'operation' }) - afterEach(() => { - delete spanContexts[0]._sampling.priority - }) - - it('should not sample anything when trace is kept', done => { - sampler = new SpanSampler({}) - try { - const ingested = sampler.sample(spanContexts[0]) - expect(ingested).to.be.undefined - done() - } catch (err) { done(err) } - }) + try { + const ingested = sampler.sample(spanContext) + expect(ingested).to.be.undefined + done() + } catch (err) { done(err) } }) - describe('rules match properly', () => { - it('should properly sample a single span', () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: 'operation', - sampleRate: 1.0, - maxPerSecond: 5 - } - ] - }) - - sampler.sample(spanContexts[0]) - expect(spans[0].context()._spanSampling).to.eql({ - sampleRate: 1.0, - maxPerSecond: 5 - }) - expect(spans[1].context()._spanSampling).to.be.undefined - expect(spans[2].context()._spanSampling).to.be.undefined - expect(spans[3].context()._spanSampling).to.be.undefined - expect(spans[4].context()._spanSampling).to.be.undefined - expect(spans[5].context()._spanSampling).to.be.undefined - expect(spans[6].context()._spanSampling).to.be.undefined - expect(spans[7].context()._spanSampling).to.be.undefined - expect(spans[8].context()._spanSampling).to.be.undefined + it('adds _spanSampling when sampled successfully', () => { + const sampler = new SpanSampler({ + spanSamplingRules: [ + { + service: 'test', + name: 'operation', + sampleRate: 1.0, + maxPerSecond: 5 + } + ] }) - it('should consider missing service as match-all for service name', () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - name: 'sub_second_operation_*', - sampleRate: 1.0, - maxPerSecond: 5 - } - ] - }) - - const spanSampling = { - sampleRate: 1.0, - maxPerSecond: 5 - } - sampler.sample(spanContexts[0]) - expect(spans[0].context()._spanSampling).to.be.undefined - expect(spans[1].context()._spanSampling).to.be.undefined - expect(spans[2].context()._spanSampling).to.be.undefined - // Only 3 and 4 should match because of the name pattern - expect(spans[3].context()._spanSampling).to.eql(spanSampling) - expect(spans[4].context()._spanSampling).to.eql(spanSampling) - expect(spans[5].context()._spanSampling).to.be.undefined - expect(spans[6].context()._spanSampling).to.be.undefined - expect(spans[7].context()._spanSampling).to.be.undefined - expect(spans[8].context()._spanSampling).to.be.undefined + const spanContext = { + _spanId: id('1234567812345678'), + _sampling: {}, + _trace: { + started: [] + }, + _name: 'operation', + _tags: {} + } + spanContext._trace.started.push({ + context: sinon.stub().returns(spanContext), + tracer: sinon.stub().returns({ + _service: 'test' + }), + _name: 'operation' }) - it('should consider missing name as match-all for span name', () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - sampleRate: 1.0, - maxPerSecond: 10 - } - ] - }) - - const spanSampling = { - sampleRate: 1.0, - maxPerSecond: 10 - } - sampler.sample(spanContexts[0]) - expect(spans[0].context()._spanSampling).to.eql(spanSampling) - expect(spans[1].context()._spanSampling).to.eql(spanSampling) - expect(spans[2].context()._spanSampling).to.eql(spanSampling) - expect(spans[3].context()._spanSampling).to.eql(spanSampling) - expect(spans[4].context()._spanSampling).to.eql(spanSampling) - expect(spans[5].context()._spanSampling).to.eql(spanSampling) - expect(spans[8].context()._spanSampling).to.eql(spanSampling) - // Should not match because of different service name - expect(spans[6].context()._spanSampling).to.be.undefined - expect(spans[7].context()._spanSampling).to.be.undefined - }) + sampler.sample(spanContext) - it('should stop at first rule match', () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: 'operation', - sampleRate: 1.0, - maxPerSecond: 5 - }, - { - service: 'test', - name: 'operation', - sampleRate: 1.0, - maxPerSecond: 10 - } - ] - }) + expect(spies.match).to.be.called + expect(spies.sample).to.be.called + expect(spies.sampleRate.get).to.be.called + expect(spies.maxPerSecond.get).to.be.called - sampler.sample(spanContexts[0]) - expect(spans[0].context()._spanSampling).to.eql({ - sampleRate: 1.0, - maxPerSecond: 5 - }) - expect(spans[1].context()._spanSampling).to.be.undefined - expect(spans[2].context()._spanSampling).to.be.undefined - expect(spans[3].context()._spanSampling).to.be.undefined - expect(spans[4].context()._spanSampling).to.be.undefined - expect(spans[5].context()._spanSampling).to.be.undefined - expect(spans[6].context()._spanSampling).to.be.undefined - expect(spans[7].context()._spanSampling).to.be.undefined - expect(spans[8].context()._spanSampling).to.be.undefined + expect(spanContext._spanSampling).to.eql({ + sampleRate: 1.0, + maxPerSecond: 5 }) + }) - it('should use span service name tags where present', () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'span-service', - sampleRate: 1.0, - maxPerSecond: 5 - } - ] - }) - - const spanSampling = { - sampleRate: 1.0, - maxPerSecond: 5 - } - sampler.sample(spanContexts[0]) - expect(spans[0].context()._spanSampling).to.be.undefined - expect(spans[1].context()._spanSampling).to.be.undefined - expect(spans[2].context()._spanSampling).to.be.undefined - expect(spans[3].context()._spanSampling).to.be.undefined - expect(spans[4].context()._spanSampling).to.be.undefined - expect(spans[5].context()._spanSampling).to.be.undefined - expect(spans[6].context()._spanSampling).to.eql(spanSampling) - expect(spans[7].context()._spanSampling).to.eql(spanSampling) - expect(spans[8].context()._spanSampling).to.be.undefined + it('should stop at first rule match', () => { + const sampler = new SpanSampler({ + spanSamplingRules: [ + { + service: 'does-not-match', + name: 'operation', + sampleRate: 1.0, + maxPerSecond: 3 + }, + { + service: 'test', + name: 'operation', + sampleRate: 1.0, + maxPerSecond: 5 + }, + { + service: 'test', + name: 'operation', + sampleRate: 1.0, + maxPerSecond: 10 + } + ] }) - it('should properly sample multiple single spans with one rule', () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: '*_2', - sampleRate: 1.0, - maxPerSecond: 5 - } - ] - }) - - const spanSampling = { - sampleRate: 1.0, - maxPerSecond: 5 - } - sampler.sample(spanContexts[0]) - expect(spans[1].context()._spanSampling).to.be.undefined - expect(spans[2].context()._spanSampling).to.be.undefined - expect(spans[3].context()._spanSampling).to.be.undefined - expect(spans[4].context()._spanSampling).to.eql(spanSampling) - expect(spans[5].context()._spanSampling).to.eql(spanSampling) - expect(spans[6].context()._spanSampling).to.be.undefined - expect(spans[7].context()._spanSampling).to.be.undefined - expect(spans[8].context()._spanSampling).to.be.undefined + const spanContext = { + _spanId: id('1234567812345678'), + _sampling: {}, + _trace: { + started: [] + }, + _name: 'operation', + _tags: {} + } + spanContext._trace.started.push({ + context: sinon.stub().returns(spanContext), + tracer: sinon.stub().returns({ + _service: 'test' + }), + _name: 'operation' }) - it('should properly sample mutiple single spans with multiple rules', () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: 'operation', - sampleRate: 1.0, - maxPerSecond: 5 - }, - { - service: 'test', - name: '*_second*', - sampleRate: 1.0, - maxPerSecond: 10 - } - ] - }) - - sampler.sample(spanContexts[0]) - expect(spans[0].context()._spanSampling, { - sampleRate: 1.0, - maxPerSecond: 5 - }) - expect(spans[3].context()._spanSampling, { - sampleRate: 1.0, - maxPerSecond: 10 - }) - expect(spans[4].context()._spanSampling, { - sampleRate: 1.0, - maxPerSecond: 10 - }) - expect(spans[5].context()._spanSampling, { - sampleRate: 1.0, - maxPerSecond: 10 - }) - }) + sampler.sample(spanContext) - it('should properly sample renamed spans', () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: 'renamed', - sampleRate: 1.0, - maxPerSecond: 1 - } - ] - }) + expect(spies.match).to.be.called + expect(spies.sample).to.be.called + expect(spies.sampleRate.get).to.be.called + expect(spies.maxPerSecond.get).to.be.called - sampler.sample(spanContexts[0]) - expect(spans[0].context()._spanSampling).to.be.undefined - expect(spans[1].context()._spanSampling).to.be.undefined - expect(spans[2].context()._spanSampling).to.be.undefined - expect(spans[3].context()._spanSampling).to.be.undefined - expect(spans[4].context()._spanSampling).to.be.undefined - expect(spans[5].context()._spanSampling).to.be.undefined - expect(spans[6].context()._spanSampling).to.be.undefined - expect(spans[7].context()._spanSampling).to.be.undefined - expect(spans[8].context()._spanSampling).to.eql({ - sampleRate: 1.0, - maxPerSecond: 1 - }) + expect(spanContext._spanSampling).to.eql({ + sampleRate: 1.0, + maxPerSecond: 5 }) }) - describe('sampleRate', () => { - beforeEach(() => { - sinon.stub(Math, 'random') + it('should sample multiple spans with one rule', () => { + const sampler = new SpanSampler({ + spanSamplingRules: [ + { + service: 'test', + name: '*operation', + sampleRate: 1.0, + maxPerSecond: 5 + } + ] }) - afterEach(() => { - Math.random.restore() - }) - - it('should sample a matched span on allowed sample rate', () => { - Math.random.returns(0.5) - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: 'operation', - sampleRate: 1.0, - maxPerSecond: 1 - } - ] - }) + // Create two span contexts + const started = [] + const firstSpanContext = { + _spanId: id('1234567812345678'), + _sampling: {}, + _trace: { + started + }, + _name: 'operation', + _tags: {} + } + const secondSpanContext = { + ...firstSpanContext, + _spanId: id('1234567812345679'), + _name: 'second operation' + } - sampler.sample(spanContexts[0]) - expect(spans[0].context()).to.haveOwnProperty('_spanSampling') + // Add spans for both to the context + started.push({ + context: sinon.stub().returns(firstSpanContext), + tracer: sinon.stub().returns({ + _service: 'test' + }), + _name: 'operation' }) - - it('should not sample a matched span on non-allowed sample rate', () => { - Math.random.returns(0.5) - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: 'operation', - sampleRate: 0.3, - maxPerSecond: 1 - } - ] - }) - - sampler.sample(spanContexts[0]) - for (const span of spans) { - expect(span.context()).to.not.haveOwnProperty('_spanSampling') - } + started.push({ + context: sinon.stub().returns(secondSpanContext), + tracer: sinon.stub().returns({ + _service: 'test' + }), + _name: 'operation' }) - it('should selectively sample based on sample rates', () => { - Math.random.returns(0.5) - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: 'operation', - sampleRate: 0.3, - maxPerSecond: 1 - }, - { - service: 'test', - name: 'second_operation', - sampleRate: 1.0, - maxPerSecond: 1 - } - ] - }) - - sampler.sample(spanContexts[0]) - expect(spans[2].context()).to.haveOwnProperty('_spanSampling') - expect(spans[0].context()).to.not.haveOwnProperty('_spanSampling') - }) - }) + sampler.sample(firstSpanContext) - describe('maxPerSecond', () => { - it('should not create limiter without finite maxPerSecond', () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: 'operation', - sampleRate: 1.0 - } - ] - }) + expect(spies.match).to.be.called + expect(spies.sample).to.be.called + expect(spies.sampleRate.get).to.be.called + expect(spies.maxPerSecond.get).to.be.called - const rule = sampler._rules[0] - expect(rule._limiter).to.equal(undefined) - expect(rule.maxPerSecond).to.equal(undefined) + expect(firstSpanContext._spanSampling).to.eql({ + sampleRate: 1.0, + maxPerSecond: 5 }) - - it('should create limiter with finite maxPerSecond', () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: 'operation', - sampleRate: 1.0, - maxPerSecond: 123 - } - ] - }) - - const rule = sampler._rules[0] - expect(rule._limiter).to.not.equal(undefined) - expect(rule).to.have.property('maxPerSecond', 123) + expect(secondSpanContext._spanSampling).to.eql({ + sampleRate: 1.0, + maxPerSecond: 5 }) + }) - it('should not sample spans past the rate limit', () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: 'operation', - sampleRate: 1.0, - maxPerSecond: 1 - } - ] - }) - - sampler.sample(spanContexts[0]) - expect(spans[0].context()).to.haveOwnProperty('_spanSampling') - delete spans[0].context()._spanSampling - - // with how quickly these tests execute, the limiter should not allow the - // next call to sample any spans - sampler.sample(spanContexts[0]) - expect(spans[0].context()).to.not.haveOwnProperty('_spanSampling') + it('should sample mutiple spans with multiple rules', () => { + const sampler = new SpanSampler({ + spanSamplingRules: [ + { + service: 'test', + name: 'operation', + sampleRate: 1.0, + maxPerSecond: 5 + }, + { + service: 'test', + name: 'second*', + sampleRate: 1.0, + maxPerSecond: 3 + } + ] }) - it('should map different rules to different rate limiters', () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: 'operation', - sampleRate: 1.0, - maxPerSecond: 1 - }, - { - service: 'test', - name: 'sub_operation', - sampleRate: 1.0, - maxPerSecond: 2 - } - ] - }) - - sampler.sample(spanContexts[0]) - expect(spans[0].context()).to.haveOwnProperty('_spanSampling') - expect(spans[1].context()).to.haveOwnProperty('_spanSampling') - delete spans[0].context()._spanSampling - delete spans[1].context()._spanSampling + // Create two span contexts + const started = [] + const firstSpanContext = { + _spanId: id('1234567812345678'), + _sampling: {}, + _trace: { + started + }, + _name: 'operation', + _tags: {} + } + const secondSpanContext = { + ...firstSpanContext, + _spanId: id('1234567812345679'), + _name: 'second operation' + } - // with how quickly these tests execute, the limiter should not allow the - // next call to sample any spans - sampler.sample(spanContexts[0]) - expect(spans[0].context()).to.not.haveOwnProperty('_spanSampling') - expect(spans[1].context()).to.haveOwnProperty('_spanSampling') + // Add spans for both to the context + started.push({ + context: sinon.stub().returns(firstSpanContext), + tracer: sinon.stub().returns({ + _service: 'test' + }), + _name: 'operation' }) - - it('should map limit by all spans matching pattern', () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: 'sub_second_operation_*', - sampleRate: 1.0, - maxPerSecond: 3 - } - ] - }) - - // First time around both should have spanSampling to prove match - sampler.sample(spanContexts[0]) - expect(spans[3].context()).to.haveOwnProperty('_spanSampling') - expect(spans[4].context()).to.haveOwnProperty('_spanSampling') - delete spans[3].context()._spanSampling - delete spans[4].context()._spanSampling - - // Second time around only first should have spanSampling to prove limits - sampler.sample(spanContexts[0]) - expect(spans[3].context()).to.haveOwnProperty('_spanSampling') - expect(spans[4].context()).to.not.haveOwnProperty('_spanSampling') + started.push({ + context: sinon.stub().returns(secondSpanContext), + tracer: sinon.stub().returns({ + _service: 'test' + }), + _name: 'operation' }) - it('should allow unlimited rate limits', async () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: 'operation', - sampleRate: 1.0 - } - ] - }) - - const interval = setInterval(() => { - sampler.sample(spanContexts[0]) - expect(spans[0].context()).to.haveOwnProperty('_spanSampling') - delete spans[0].context()._spanSampling - }, 1) + sampler.sample(firstSpanContext) - await new Promise(resolve => { - setTimeout(resolve, 1000) - }) + expect(spies.match).to.be.called + expect(spies.sample).to.be.called + expect(spies.sampleRate.get).to.be.called + expect(spies.maxPerSecond.get).to.be.called - clearInterval(interval) + expect(firstSpanContext._spanSampling).to.eql({ + sampleRate: 1.0, + maxPerSecond: 5 }) - - it('should sample if enough time has elapsed', async () => { - sampler = new SpanSampler({ - spanSamplingRules: [ - { - service: 'test', - name: 'operation', - sampleRate: 1.0, - maxPerSecond: 1 - } - ] - }) - - await new Promise(resolve => { - sampler.sample(spanContexts[0]) - const before = spans[0].context()._spanSampling - delete spans[0].context()._spanSampling - - setTimeout(() => { - sampler.sample(spanContexts[0]) - const after = spans[0].context()._spanSampling - delete spans[0].context()._spanSampling - - expect(before).to.eql(after) - resolve() - }, 1000) - }) + expect(secondSpanContext._spanSampling).to.eql({ + sampleRate: 1.0, + maxPerSecond: 3 }) }) }) From cc2d72fbc92a9eb541efaeadcb47bdd20c304d68 Mon Sep 17 00:00:00 2001 From: Attila Szegedi Date: Thu, 4 Jan 2024 13:11:49 +0100 Subject: [PATCH 18/21] PROF-8850: Add diagnostic to flaky test (#3927) * Ignore samples without endpoints instead of asserting all will have endpoints * Emit base64-encoded pprof in an error message so we can debug flaky tests --- integration-tests/profiler.spec.js | 62 ++++++++++++++++-------------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/integration-tests/profiler.spec.js b/integration-tests/profiler.spec.js index 76685bc30be..86f968c1203 100644 --- a/integration-tests/profiler.spec.js +++ b/integration-tests/profiler.spec.js @@ -63,7 +63,7 @@ async function getLatestProfile (cwd, pattern) { .name const pprofGzipped = await fs.readFile(path.join(cwd, pprofEntry)) const pprofUnzipped = zlib.gunzipSync(pprofGzipped) - return Profile.decode(pprofUnzipped) + return { profile: Profile.decode(pprofUnzipped), encoded: pprofGzipped.toString('base64') } } async function gatherNetworkTimelineEvents (cwd, scriptFilePath, eventType, args) { @@ -81,9 +81,9 @@ async function gatherNetworkTimelineEvents (cwd, scriptFilePath, eventType, args await processExitPromise(proc, 5000) const procEnd = BigInt(Date.now() * 1000000) - const prof = await getLatestProfile(cwd, /^events_.+\.pprof$/) + const { profile, encoded } = await getLatestProfile(cwd, /^events_.+\.pprof$/) - const strings = prof.stringTable + const strings = profile.stringTable const tsKey = strings.dedup('end_timestamp_ns') const eventKey = strings.dedup('event') const hostKey = strings.dedup('host') @@ -92,7 +92,7 @@ async function gatherNetworkTimelineEvents (cwd, scriptFilePath, eventType, args const nameKey = strings.dedup('operation') const eventValue = strings.dedup(eventType) const events = [] - for (const sample of prof.sample) { + for (const sample of profile.sample) { let ts, event, host, address, port, name for (const label of sample.label) { switch (label.key) { @@ -102,18 +102,18 @@ async function gatherNetworkTimelineEvents (cwd, scriptFilePath, eventType, args case hostKey: host = label.str; break case addressKey: address = label.str; break case portKey: port = label.num; break - default: assert.fail(`Unexpected label key ${label.key} ${strings.strings[label.key]}`) + default: assert.fail(`Unexpected label key ${label.key} ${strings.strings[label.key]} ${encoded}`) } } // Timestamp must be defined and be between process start and end time - assert.isDefined(ts) - assert.isTrue(ts <= procEnd) - assert.isTrue(ts >= procStart) + assert.isDefined(ts, encoded) + assert.isTrue(ts <= procEnd, encoded) + assert.isTrue(ts >= procStart, encoded) // Gather only DNS events; ignore sporadic GC events if (event === eventValue) { - assert.isDefined(name) + assert.isDefined(name, encoded) // Exactly one of these is defined - assert.isTrue(!!address !== !!host) + assert.isTrue(!!address !== !!host, encoded) const ev = { name: strings.strings[name] } if (address) { ev.address = strings.strings[address] @@ -169,7 +169,7 @@ describe('profiler', () => { await processExitPromise(proc, 5000) const procEnd = BigInt(Date.now() * 1000000) - const prof = await getLatestProfile(cwd, /^wall_.+\.pprof$/) + const { profile, encoded } = await getLatestProfile(cwd, /^wall_.+\.pprof$/) // We check the profile for following invariants: // - every sample needs to have an 'end_timestamp_ns' label that has values (nanos since UNIX @@ -182,7 +182,7 @@ describe('profiler', () => { const rootSpans = new Set() const endpoints = new Set() const spans = new Map() - const strings = prof.stringTable + const strings = profile.stringTable const tsKey = strings.dedup('end_timestamp_ns') const spanKey = strings.dedup('span id') const rootSpanKey = strings.dedup('local root span id') @@ -191,7 +191,7 @@ describe('profiler', () => { const threadIdKey = strings.dedup('thread id') const osThreadIdKey = strings.dedup('os thread id') const threadNameValue = strings.dedup('Main Event Loop') - for (const sample of prof.sample) { + for (const sample of profile.sample) { let ts, spanId, rootSpanId, endpoint, threadName, threadId, osThreadId for (const label of sample.label) { switch (label.key) { @@ -202,29 +202,33 @@ describe('profiler', () => { case threadNameKey: threadName = label.str; break case threadIdKey: threadId = label.str; break case osThreadIdKey: osThreadId = label.str; break - default: assert.fail(`Unexpected label key ${strings.dedup(label.key)}`) + default: assert.fail(`Unexpected label key ${strings.dedup(label.key)} ${encoded}`) } } // Timestamp must be defined and be between process start and end time - assert.isDefined(ts) - assert.isNumber(osThreadId) - assert.equal(threadId, strings.dedup('0')) - assert.isTrue(ts <= procEnd) - assert.isTrue(ts >= procStart) + assert.isDefined(ts, encoded) + assert.isNumber(osThreadId, encoded) + assert.equal(threadId, strings.dedup('0'), encoded) + assert.isTrue(ts <= procEnd, encoded) + assert.isTrue(ts >= procStart, encoded) // Thread name must be defined and exactly equal "Main Event Loop" - assert.equal(threadName, threadNameValue) + assert.equal(threadName, threadNameValue, encoded) // Either all or none of span-related labels are defined - if (spanId || rootSpanId || endpoint) { - assert.isDefined(spanId) - assert.isDefined(rootSpanId) - assert.isDefined(endpoint) + if (endpoint === undefined) { + // It is possible to catch a sample executing in tracer's startSpan so + // that endpoint is not yet set. We'll ignore those samples. + continue + } + if (spanId || rootSpanId) { + assert.isDefined(spanId, encoded) + assert.isDefined(rootSpanId, encoded) rootSpans.add(rootSpanId) const spanData = { rootSpanId, endpoint } const existingSpanData = spans.get(spanId) if (existingSpanData) { // Span's root span and endpoint must be consistent across samples - assert.deepEqual(spanData, existingSpanData) + assert.deepEqual(spanData, existingSpanData, encoded) } else { // New span id, store span data spans.set(spanId, spanData) @@ -237,16 +241,16 @@ describe('profiler', () => { endpoints.add(endpoint) break default: - assert.fail(`Unexpected endpoint value ${endpointVal}`) + assert.fail(`Unexpected endpoint value ${endpointVal} ${encoded}`) } } } } // Need to have a total of 9 different spans, with 3 different root spans // and 3 different endpoints. - assert.equal(spans.size, 9) - assert.equal(rootSpans.size, 3) - assert.equal(endpoints.size, 3) + assert.equal(spans.size, 9, encoded) + assert.equal(rootSpans.size, 3, encoded) + assert.equal(endpoints.size, 3, encoded) }) if (semver.gte(process.version, '16.0.0')) { From 63131a6530bd1d88299969f78f86a1fbf1f7df80 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Thu, 4 Jan 2024 20:39:09 +0800 Subject: [PATCH 19/21] Fix couchbase segfault. On >=3.2.0 <3.2.2 it would segfault on cluster.close() (#3926) https://issues.couchbase.com/browse/JSCBC-936 --- packages/datadog-instrumentations/src/couchbase.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/datadog-instrumentations/src/couchbase.js b/packages/datadog-instrumentations/src/couchbase.js index 386a4b676ed..0e2532251e8 100644 --- a/packages/datadog-instrumentations/src/couchbase.js +++ b/packages/datadog-instrumentations/src/couchbase.js @@ -252,9 +252,10 @@ addHook({ name: 'couchbase', file: 'lib/cluster.js', versions: ['^3.0.7', '^3.1. return Cluster }) -// semver >=3.2.0 +// semver >=3.2.2 +// NOTE: <3.2.2 segfaults on cluster.close() https://issues.couchbase.com/browse/JSCBC-936 -addHook({ name: 'couchbase', file: 'dist/collection.js', versions: ['>=3.2.0'] }, collection => { +addHook({ name: 'couchbase', file: 'dist/collection.js', versions: ['>=3.2.2'] }, collection => { const Collection = collection.Collection wrapAllNames(['upsert', 'insert', 'replace'], name => { @@ -264,7 +265,7 @@ addHook({ name: 'couchbase', file: 'dist/collection.js', versions: ['>=3.2.0'] } return collection }) -addHook({ name: 'couchbase', file: 'dist/bucket.js', versions: ['>=3.2.0'] }, bucket => { +addHook({ name: 'couchbase', file: 'dist/bucket.js', versions: ['>=3.2.2'] }, bucket => { const Bucket = bucket.Bucket shimmer.wrap(Bucket.prototype, 'collection', getCollection => { return function () { @@ -278,7 +279,7 @@ addHook({ name: 'couchbase', file: 'dist/bucket.js', versions: ['>=3.2.0'] }, bu return bucket }) -addHook({ name: 'couchbase', file: 'dist/cluster.js', versions: ['3.2.0 - 3.2.1', '>=3.2.2'] }, (cluster) => { +addHook({ name: 'couchbase', file: 'dist/cluster.js', versions: ['>=3.2.2'] }, (cluster) => { const Cluster = cluster.Cluster shimmer.wrap(Cluster.prototype, 'query', wrapV3Query) From df6c2071619c79533c562eb2025b11d3acde48ab Mon Sep 17 00:00:00 2001 From: Ayan Khan Date: Thu, 4 Jan 2024 12:21:38 -0500 Subject: [PATCH 20/21] bump up iitm version (#3929) --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 02ace1a3c20..d2532619658 100644 --- a/package.json +++ b/package.json @@ -79,7 +79,7 @@ "crypto-randomuuid": "^1.0.0", "dc-polyfill": "^0.1.2", "ignore": "^5.2.4", - "import-in-the-middle": "^1.4.2", + "import-in-the-middle": "^1.7.1", "int64-buffer": "^0.1.9", "ipaddr.js": "^2.1.0", "istanbul-lib-coverage": "3.2.0", From 0fbd24f083ffcdfb4f750094c53220caefd911b6 Mon Sep 17 00:00:00 2001 From: Ayan Khan Date: Thu, 4 Jan 2024 13:47:27 -0500 Subject: [PATCH 21/21] v4.23.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d2532619658..33e40c3010a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dd-trace", - "version": "4.22.0", + "version": "4.23.0", "description": "Datadog APM tracing client for JavaScript", "main": "index.js", "typings": "index.d.ts",