Include send-data missing headers and organize telemetry config variables (#3055)

* Include send-data missing headers

* Rename DD_TELEMETRY_DEBUG_ENABLED as DD_TELEMETRY_DIAGNOSTIC_LOG_COLLECTION_ENABLED

* Move DD_TELEMETRY_HEARTBEAT_INTERVAL to config.js

* Remove DD_TELEMETRY_DIAGNOSTIC_LOG_COLLECTION_ENABLED

Author: iunanua

packages/dd-trace/src/appsec/iast/telemetry/logs.js

@@ -28,7 +28,7 @@ function sendLogs () {
 }
 
 function isLevelEnabled (level) {
-  return isLogCollectionEnabled(config) && (level !== 'DEBUG' || config.telemetry.debug)
+  return isLogCollectionEnabled(config) && level !== 'DEBUG'
 }
 
 function isLogCollectionEnabled (config) {

packages/dd-trace/src/config.js

@@ -216,8 +216,11 @@ class Config {
       process.env.DD_TRACE_TELEMETRY_ENABLED,
       !inServerlessEnvironment
     )
-    const DD_TELEMETRY_DEBUG_ENABLED = coalesce(
-      process.env.DD_TELEMETRY_DEBUG_ENABLED,
+    const DD_TELEMETRY_HEARTBEAT_INTERVAL = process.env.DD_TELEMETRY_HEARTBEAT_INTERVAL
+      ? parseInt(process.env.DD_TELEMETRY_HEARTBEAT_INTERVAL) * 1000
+      : 60000
+    const DD_TELEMETRY_DEBUG = coalesce(
+      process.env.DD_TELEMETRY_DEBUG,
       false
     )
     const DD_TRACE_AGENT_PROTOCOL_VERSION = coalesce(
@@ -513,8 +516,9 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
     // Disabled for CI Visibility's agentless
     this.telemetry = {
       enabled: DD_TRACE_EXPORTER !== 'datadog' && isTrue(DD_TRACE_TELEMETRY_ENABLED),
+      heartbeatInterval: DD_TELEMETRY_HEARTBEAT_INTERVAL,
       logCollection: isTrue(DD_TELEMETRY_LOG_COLLECTION_ENABLED),
-      debug: isTrue(DD_TELEMETRY_DEBUG_ENABLED)
+      debug: isTrue(DD_TELEMETRY_DEBUG)
     }
     this.protocolVersion = DD_TRACE_AGENT_PROTOCOL_VERSION
     this.tagsHeaderMaxLength = parseInt(DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH)

packages/dd-trace/src/telemetry/index.js

@@ -6,10 +6,6 @@ const os = require('os')
 const dependencies = require('./dependencies')
 const { sendData } = require('./send-data')
 
-const HEARTBEAT_INTERVAL = process.env.DD_TELEMETRY_HEARTBEAT_INTERVAL
-  ? Number(process.env.DD_TELEMETRY_HEARTBEAT_INTERVAL) * 1000
-  : 60000
-
 const telemetryStartChannel = dc.channel('datadog:telemetry:start')
 const telemetryStopChannel = dc.channel('datadog:telemetry:stop')
 
@@ -19,6 +15,7 @@ let pluginManager
 let application
 let host
 let interval
+let heartbeatInterval
 const sentIntegrations = new Set()
 
 function getIntegrations () {
@@ -108,7 +105,7 @@ function createHostObject () {
 }
 
 function getTelemetryData () {
-  return { config, application, host, heartbeatInterval: HEARTBEAT_INTERVAL }
+  return { config, application, host, heartbeatInterval }
 }
 
 function start (aConfig, thePluginManager) {
@@ -119,11 +116,13 @@ function start (aConfig, thePluginManager) {
   pluginManager = thePluginManager
   application = createAppObject()
   host = createHostObject()
+  heartbeatInterval = config.telemetry.heartbeatInterval
+
   dependencies.start(config, application, host)
   sendData(config, application, host, 'app-started', appStarted())
   interval = setInterval(() => {
     sendData(config, application, host, 'app-heartbeat')
-  }, HEARTBEAT_INTERVAL)
+  }, heartbeatInterval)
   interval.unref()
   process.on('beforeExit', onBeforeExit)
 

packages/dd-trace/src/telemetry/send-data.js

@@ -1,4 +1,20 @@
 const request = require('../exporters/common/request')
+
+function getHeaders (config, application, reqType) {
+  const headers = {
+    'content-type': 'application/json',
+    'dd-telemetry-api-version': 'v1',
+    'dd-telemetry-request-type': reqType,
+    'dd-client-library-language': application.language_name,
+    'dd-client-library-version': application.tracer_version
+  }
+  const debug = config.telemetry && config.telemetry.debug
+  if (debug) {
+    headers['dd-telemetry-debug-enabled'] = 'true'
+  }
+  return headers
+}
+
 let seqId = 0
 
 function getPayload (payload) {
@@ -25,11 +41,7 @@ function sendData (config, application, host, reqType, payload = {}) {
     port,
     method: 'POST',
     path: '/telemetry/proxy/api/v2/apmtelemetry',
-    headers: {
-      'content-type': 'application/json',
-      'dd-telemetry-api-version': 'v1',
-      'dd-telemetry-request-type': reqType
-    }
+    headers: getHeaders(config, application, reqType)
   }
   const data = JSON.stringify({
     api_version: 'v1',

packages/dd-trace/test/appsec/iast/iast-log.spec.js

@@ -6,20 +6,10 @@ const ddBasePath = calculateDDBasePath(__dirname)
 const EOL = '\n'
 
 describe('IAST log', () => {
-  const telemetryDebugConfig = {
-    config: {
-      telemetry: {
-        logCollection: true,
-        debug: true
-      }
-    }
-  }
-
   const telemetryDefaultConfig = {
     config: {
       telemetry: {
-        logCollection: true,
-        debug: false
+        logCollection: true
       }
     }
   }
@@ -77,81 +67,6 @@ describe('IAST log', () => {
     telemetryLogs.stop()
   })
 
-  describe('debug', () => {
-    it('should call log.debug', () => {
-      iastLog.debug('debug')
-
-      expect(log.debug).to.be.calledOnceWith('debug')
-    })
-
-    it('should call log.debug and not publish msg via telemetry', () => {
-      iastLog.debugAndPublish('debug')
-
-      expect(log.debug).to.be.calledOnceWith('debug')
-      expect(telemetryLogs.publish).to.not.be.called
-    })
-
-    it('should call log.debug and publish msg via telemetry', () => {
-      onTelemetryStart(telemetryDebugConfig)
-
-      iastLog.debugAndPublish('debug')
-
-      expect(log.debug).to.be.calledOnceWith('debug')
-      expect(telemetryLogs.publish).to.be.calledOnceWith({ message: 'debug', level: 'DEBUG' })
-    })
-
-    it('should chain multiple debug calls', () => {
-      onTelemetryStart(telemetryDebugConfig)
-
-      iastLog.debug('debug').debugAndPublish('debugAndPublish').debug('debug2')
-
-      expect(log.debug).to.be.calledThrice
-      expect(log.debug.getCall(0).args[0]).to.be.eq('debug')
-      expect(log.debug.getCall(1).args[0]).to.be.eq('debugAndPublish')
-      expect(log.debug.getCall(2).args[0]).to.be.eq('debug2')
-      expect(telemetryLogs.publish).to.be.calledOnceWith({ message: 'debugAndPublish', level: 'DEBUG' })
-    })
-
-    it('should chain multiple debug calls', () => {
-      onTelemetryStart(telemetryDebugConfig)
-
-      iastLog.debug('debug')
-
-      telemetryLogs.stop()
-
-      iastLog.debugAndPublish('debugAndPublish').debug('debug2')
-    })
-  })
-
-  describe('info', () => {
-    it('should call log.info', () => {
-      iastLog.info('info')
-
-      expect(log.info).to.be.calledOnceWith('info')
-    })
-
-    it('should call log.info and publish msg via telemetry', () => {
-      onTelemetryStart(telemetryDebugConfig)
-
-      iastLog.infoAndPublish('info')
-
-      expect(log.info).to.be.calledOnceWith('info')
-      expect(telemetryLogs.publish).to.be.calledOnceWith({ message: 'info', level: 'DEBUG' })
-    })
-
-    it('should chain multiple info calls', () => {
-      onTelemetryStart(telemetryDebugConfig)
-
-      iastLog.info('info').infoAndPublish('infoAndPublish').info('info2')
-
-      expect(log.info).to.be.calledThrice
-      expect(log.info.getCall(0).args[0]).to.be.eq('info')
-      expect(log.info.getCall(1).args[0]).to.be.eq('infoAndPublish')
-      expect(log.info.getCall(2).args[0]).to.be.eq('info2')
-      expect(telemetryLogs.publish).to.be.calledOnceWith({ message: 'infoAndPublish', level: 'DEBUG' })
-    })
-  })
-
   describe('warn', () => {
     it('should call log.warn', () => {
       iastLog.warn('warn')

packages/dd-trace/test/appsec/iast/telemetry/logs.spec.js

@@ -131,32 +131,7 @@ describe('telemetry logs', () => {
   })
 
   describe('sendData', () => {
-    const app = {}
-    const host = {}
-
-    it('should be called with DEBUG level and error if config.telemetry.debug = true', () => {
-      const logCollectorAdd = sinon.stub()
-      const logs = proxyquire('../../../../src/appsec/iast/telemetry/logs', {
-        '../../../../../diagnostics_channel': dc,
-        './log_collector': {
-          add: logCollectorAdd
-        }
-      })
-      logs.start()
-
-      onTelemetryStartMsg.config.telemetry.debug = true
-      onTelemetryStartMsg.application = app
-      onTelemetryStartMsg.host = host
-      onTelemetryStart()(onTelemetryStartMsg)
-
-      const error = new Error('test')
-      const stack = error.stack
-      logs.publish({ message: error.message, stack_trace: stack, level: 'DEBUG' })
-
-      expect(logCollectorAdd).to.be.calledOnceWith(match({ message: 'test', level: 'DEBUG', stack_trace: stack }))
-    })
-
-    it('should be not called with DEBUG level if config.telemetry.debug = false', () => {
+    it('should be not called with DEBUG level', () => {
       const logCollectorAdd = sinon.stub()
       const logs = proxyquire('../../../../src/appsec/iast/telemetry/logs', {
         '../../../../../diagnostics_channel': dc,

packages/dd-trace/test/config.spec.js

@@ -822,10 +822,22 @@ describe('Config', () => {
 
     expect(config.telemetry).to.not.be.undefined
     expect(config.telemetry.enabled).to.be.true
+    expect(config.telemetry.heartbeatInterval).to.eq(60000)
     expect(config.telemetry.logCollection).to.be.false
     expect(config.telemetry.debug).to.be.false
   })
 
+  it('should set DD_TELEMETRY_HEARTBEAT_INTERVAL', () => {
+    const origTelemetryHeartbeatIntervalValue = process.env.DD_TELEMETRY_HEARTBEAT_INTERVAL
+    process.env.DD_TELEMETRY_HEARTBEAT_INTERVAL = '42'
+
+    const config = new Config()
+
+    expect(config.telemetry.heartbeatInterval).to.eq(42000)
+
+    process.env.DD_TELEMETRY_HEARTBEAT_INTERVAL = origTelemetryHeartbeatIntervalValue
+  })
+
   it('should not set DD_TRACE_TELEMETRY_ENABLED', () => {
     const origTraceTelemetryValue = process.env.DD_TRACE_TELEMETRY_ENABLED
     process.env.DD_TRACE_TELEMETRY_ENABLED = 'false'
@@ -859,15 +871,15 @@ describe('Config', () => {
     process.env.DD_IAST_ENABLED = origIastEnabledValue
   })
 
-  it('should set DD_TELEMETRY_DEBUG_ENABLED', () => {
-    const origTelemetryDebugValue = process.env.DD_TELEMETRY_DEBUG_ENABLED
-    process.env.DD_TELEMETRY_DEBUG_ENABLED = 'true'
+  it('should set DD_TELEMETRY_DEBUG', () => {
+    const origTelemetryDebugValue = process.env.DD_TELEMETRY_DEBUG
+    process.env.DD_TELEMETRY_DEBUG = 'true'
 
     const config = new Config()
 
     expect(config.telemetry.debug).to.be.true
 
-    process.env.DD_TELEMETRY_DEBUG_ENABLED = origTelemetryDebugValue
+    process.env.DD_TELEMETRY_DEBUG = origTelemetryDebugValue
   })
 
   it('should not set DD_REMOTE_CONFIGURATION_ENABLED if AWS_LAMBDA_FUNCTION_NAME is present', () => {

packages/dd-trace/test/telemetry/index.spec.js

@@ -64,7 +64,7 @@ describe('telemetry', () => {
     circularObject.child.parent = circularObject
 
     telemetry.start({
-      telemetry: { enabled: true },
+      telemetry: { enabled: true, heartbeatInterval: 60000 },
       hostname: 'localhost',
       port: traceAgent.address().port,
       service: 'test service',
@@ -154,7 +154,7 @@ describe('telemetry', () => {
       expect.fail('server should not be called')
     }).listen(0, () => {
       telemetry.start({
-        telemetry: { enabled: false },
+        telemetry: { enabled: false, heartbeatInterval: 60000 },
         hostname: 'localhost',
         port: server.address().port
       })
@@ -169,8 +169,6 @@ describe('telemetry', () => {
 
 describe('telemetry with interval change', () => {
   it('should set the interval correctly', (done) => {
-    process.env.DD_TELEMETRY_HEARTBEAT_INTERVAL = 12345
-
     const telemetry = proxyquire('../../src/telemetry', {
       '../exporters/common/docker': {
         id () {
@@ -190,7 +188,7 @@ describe('telemetry with interval change', () => {
     }
 
     telemetry.start({
-      telemetry: { enabled: true },
+      telemetry: { enabled: true, heartbeatInterval: 12345000 },
       hostname: 'localhost',
       port: 8126,
       service: 'test service',
@@ -205,7 +203,6 @@ describe('telemetry with interval change', () => {
 
     process.nextTick(() => {
       expect(intervalSetCorrectly).to.be.true
-      delete process.env.DD_TELEMETRY_HEARTBEAT_INTERVAL
       done()
     })
   })