From d28f33ed45f2ed5497d628dcc39a0f8e96ecc656 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Mon, 5 Feb 2024 17:33:17 +0100 Subject: [PATCH] [ci-visibility] Add get known tests request (#4015) --- .../early-flake-detection/get-known-tests.js | 83 +++++++++++ .../exporters/ci-visibility-exporter.js | 59 +++++++- .../requests/get-library-configuration.js | 9 +- packages/dd-trace/src/config.js | 6 + packages/dd-trace/src/plugins/ci_plugin.js | 12 ++ .../exporters/ci-visibility-exporter.spec.js | 135 +++++++++++++++++- packages/dd-trace/test/config.spec.js | 10 ++ 7 files changed, 309 insertions(+), 5 deletions(-) create mode 100644 packages/dd-trace/src/ci-visibility/early-flake-detection/get-known-tests.js diff --git a/packages/dd-trace/src/ci-visibility/early-flake-detection/get-known-tests.js b/packages/dd-trace/src/ci-visibility/early-flake-detection/get-known-tests.js new file mode 100644 index 00000000000..e9df9daa04c --- /dev/null +++ b/packages/dd-trace/src/ci-visibility/early-flake-detection/get-known-tests.js @@ -0,0 +1,83 @@ +const request = require('../../exporters/common/request') +const id = require('../../id') +const log = require('../../log') + +function getKnownTests ({ + url, + isEvpProxy, + evpProxyPrefix, + isGzipCompatible, + env, + service, + repositoryUrl, + sha, + osVersion, + osPlatform, + osArchitecture, + runtimeName, + runtimeVersion, + custom +}, done) { + const options = { + path: '/api/v2/ci/libraries/tests', + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + timeout: 20000, + url + } + + if (isGzipCompatible) { + options.headers['accept-encoding'] = 'gzip' + } + + if (isEvpProxy) { + options.path = `${evpProxyPrefix}/api/v2/ci/libraries/tests` + options.headers['X-Datadog-EVP-Subdomain'] = 'api' + } else { + const apiKey = process.env.DATADOG_API_KEY || process.env.DD_API_KEY + if (!apiKey) { + return done(new Error('Skippable suites were not fetched because Datadog API key is not defined.')) + } + + options.headers['dd-api-key'] = apiKey + } + + const data = JSON.stringify({ + data: { + id: id().toString(10), + type: 'ci_app_libraries_tests_request', + attributes: { + configurations: { + 'os.platform': osPlatform, + 'os.version': osVersion, + 'os.architecture': osArchitecture, + 'runtime.name': runtimeName, + 'runtime.version': runtimeVersion, + custom + }, + service, + env, + repository_url: repositoryUrl, + sha + } + } + }) + + request(data, options, (err, res) => { + if (err) { + done(err) + } else { + try { + const { data: { attributes: { test_full_names: knownTests } } } = JSON.parse(res) + log.debug(() => `Number of received known tests: ${Object.keys(knownTests).length}`) + done(null, knownTests) + } catch (err) { + done(err) + } + } + }) +} + +module.exports = { getKnownTests } diff --git a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js index 58e4495c6b2..28c7e79744c 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js +++ b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js @@ -5,6 +5,7 @@ const URL = require('url').URL const { sendGitMetadata: sendGitMetadataRequest } = require('./git/git_metadata') const { getLibraryConfiguration: getLibraryConfigurationRequest } = require('../requests/get-library-configuration') const { getSkippableSuites: getSkippableSuitesRequest } = require('../intelligent-test-runner/get-skippable-suites') +const { getKnownTests: getKnownTestsRequest } = require('../early-flake-detection/get-known-tests') const log = require('../../log') const AgentInfoExporter = require('../../exporters/common/agent-info-exporter') @@ -79,6 +80,13 @@ class CiVisibilityExporter extends AgentInfoExporter { this._libraryConfig?.isSuitesSkippingEnabled) } + shouldRequestKnownTests () { + return !!( + this._config.isEarlyFlakeDetectionEnabled && + this._libraryConfig?.isEarlyFlakeDetectionEnabled + ) + } + shouldRequestLibraryConfiguration () { return this._config.isIntelligentTestRunnerEnabled } @@ -116,6 +124,30 @@ class CiVisibilityExporter extends AgentInfoExporter { }) } + getKnownTests (testConfiguration, callback) { + if (!this.shouldRequestKnownTests()) { + return callback(null) + } + this._canUseCiVisProtocolPromise.then((canUseCiVisProtocol) => { + if (!canUseCiVisProtocol) { + return callback( + new Error('Known tests can not be requested because CI Visibility protocol can not be used') + ) + } + const configuration = { + url: this._getApiUrl(), + env: this._config.env, + service: this._config.service, + isEvpProxy: !!this._isUsingEvpProxy, + evpProxyPrefix: this.evpProxyPrefix, + custom: getTestConfigurationTags(this._config.tags), + isGzipCompatible: this._isGzipCompatible, + ...testConfiguration + } + getKnownTestsRequest(configuration, callback) + }) + } + /** * We can't request library configuration until we know whether we can use the * CI Visibility Protocol, hence the this._canUseCiVisProtocol promise. @@ -145,7 +177,7 @@ class CiVisibilityExporter extends AgentInfoExporter { * where the tests run in a subprocess, like Jest, * because `getLibraryConfiguration` is called only once in the main process. */ - this._libraryConfig = libraryConfig + this._libraryConfig = this.getConfiguration(libraryConfig) if (err) { callback(err, {}) @@ -156,8 +188,8 @@ class CiVisibilityExporter extends AgentInfoExporter { return callback(gitUploadError, {}) } getLibraryConfigurationRequest(configuration, (err, finalLibraryConfig) => { - this._libraryConfig = finalLibraryConfig - callback(err, finalLibraryConfig) + this._libraryConfig = this.getConfiguration(finalLibraryConfig) + callback(err, this._libraryConfig) }) }) } else { @@ -167,6 +199,27 @@ class CiVisibilityExporter extends AgentInfoExporter { }) } + // Takes into account potential kill switches + getConfiguration (remoteConfiguration) { + if (!remoteConfiguration) { + return {} + } + const { + isCodeCoverageEnabled, + isSuitesSkippingEnabled, + isItrEnabled, + requireGit, + isEarlyFlakeDetectionEnabled + } = remoteConfiguration + return { + isCodeCoverageEnabled, + isSuitesSkippingEnabled, + isItrEnabled, + requireGit, + isEarlyFlakeDetectionEnabled: isEarlyFlakeDetectionEnabled && this._config.isEarlyFlakeDetectionEnabled + } + } + sendGitMetadata (repositoryUrl) { if (!this._config.isGitUploadEnabled) { return diff --git a/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js b/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js index ea9340f4224..61ed0e70ca1 100644 --- a/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js +++ b/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js @@ -94,7 +94,14 @@ function getLibraryConfiguration ({ } } = JSON.parse(res) - const settings = { isCodeCoverageEnabled, isSuitesSkippingEnabled, isItrEnabled, requireGit } + const settings = { + isCodeCoverageEnabled, + isSuitesSkippingEnabled, + isItrEnabled, + requireGit, + // TODO: change to backend response + isEarlyFlakeDetectionEnabled: false + } log.debug(() => `Remote settings: ${JSON.stringify(settings)}`) diff --git a/packages/dd-trace/src/config.js b/packages/dd-trace/src/config.js index 7b992e31194..7f0627bb655 100644 --- a/packages/dd-trace/src/config.js +++ b/packages/dd-trace/src/config.js @@ -172,6 +172,11 @@ class Config { false ) + const DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED = coalesce( + process.env.DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED, + true + ) + const DD_TRACE_MEMCACHED_COMMAND_ENABLED = coalesce( process.env.DD_TRACE_MEMCACHED_COMMAND_ENABLED, false @@ -666,6 +671,7 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?) this.gitMetadataEnabled = isTrue(DD_TRACE_GIT_METADATA_ENABLED) this.isManualApiEnabled = this.isCiVisibility && isTrue(DD_CIVISIBILITY_MANUAL_API_ENABLED) + this.isEarlyFlakeDetectionEnabled = this.isCiVisibility && isTrue(DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED) this.openaiSpanCharLimit = DD_OPENAI_SPAN_CHAR_LIMIT diff --git a/packages/dd-trace/src/plugins/ci_plugin.js b/packages/dd-trace/src/plugins/ci_plugin.js index 10ba66caff2..e8565bfe174 100644 --- a/packages/dd-trace/src/plugins/ci_plugin.js +++ b/packages/dd-trace/src/plugins/ci_plugin.js @@ -115,6 +115,18 @@ module.exports = class CiPlugin extends Plugin { }) this.telemetry.count(TELEMETRY_ITR_SKIPPED, { testLevel: 'suite' }, skippedSuites.length) }) + + this.addSub(`ci:${this.constructor.id}:known-tests`, ({ onDone }) => { + if (!this.tracer._exporter?.getKnownTests) { + return onDone({ err: new Error('CI Visibility was not initialized correctly') }) + } + this.tracer._exporter.getKnownTests(this.testConfiguration, (err, knownTests) => { + if (err) { + log.error(`Known tests could not be fetched. ${err.message}`) + } + onDone({ err, knownTests }) + }) + }) } get telemetry () { diff --git a/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js b/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js index db9da2db76d..0fa82baba52 100644 --- a/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js +++ b/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js @@ -170,7 +170,8 @@ describe('CI Visibility Exporter', () => { requireGit: false, isCodeCoverageEnabled: true, isItrEnabled: true, - isSuitesSkippingEnabled: true + isSuitesSkippingEnabled: true, + isEarlyFlakeDetectionEnabled: false }) expect(err).not.to.exist expect(scope.isDone()).to.be.true @@ -644,4 +645,136 @@ describe('CI Visibility Exporter', () => { }) }) }) + + describe('getKnownTests', () => { + context('if early flake detection is disabled', () => { + it('should resolve immediately to undefined', (done) => { + const scope = nock(`http://localhost:${port}`) + .post('/api/v2/ci/libraries/tests') + .reply(200) + + const ciVisibilityExporter = new CiVisibilityExporter({ port, isEarlyFlakeDetectionEnabled: false }) + + ciVisibilityExporter._resolveCanUseCiVisProtocol(true) + + ciVisibilityExporter.getKnownTests({}, (err, knownTests) => { + expect(err).to.be.null + expect(knownTests).to.eql(undefined) + expect(scope.isDone()).not.to.be.true + done() + }) + }) + }) + context('if early flake detection is enabled but can not use CI Visibility protocol', () => { + it('should raise an error', (done) => { + const scope = nock(`http://localhost:${port}`) + .post('/api/v2/ci/libraries/tests') + .reply(200) + + const ciVisibilityExporter = new CiVisibilityExporter({ port, isEarlyFlakeDetectionEnabled: true }) + + ciVisibilityExporter._resolveCanUseCiVisProtocol(false) + ciVisibilityExporter._libraryConfig = { isEarlyFlakeDetectionEnabled: true } + ciVisibilityExporter.getKnownTests({}, (err) => { + expect(err.message).to.include( + 'Known tests can not be requested because CI Visibility protocol can not be used' + ) + expect(scope.isDone()).not.to.be.true + done() + }) + }) + }) + context('if early flake detection is enabled and can use CI Vis Protocol', () => { + it('should request known tests', (done) => { + const scope = nock(`http://localhost:${port}`) + .post('/api/v2/ci/libraries/tests') + .reply(200, JSON.stringify({ + data: { + attributes: { + test_full_names: ['suite1.test1', 'suite2.test2'] + } + } + })) + + const ciVisibilityExporter = new CiVisibilityExporter({ port, isEarlyFlakeDetectionEnabled: true }) + + ciVisibilityExporter._resolveCanUseCiVisProtocol(true) + ciVisibilityExporter._libraryConfig = { isEarlyFlakeDetectionEnabled: true } + ciVisibilityExporter.getKnownTests({}, (err, knownTests) => { + expect(err).to.be.null + expect(knownTests).to.eql(['suite1.test1', 'suite2.test2']) + expect(scope.isDone()).to.be.true + done() + }) + }) + it('should return an error if the request fails', (done) => { + const scope = nock(`http://localhost:${port}`) + .post('/api/v2/ci/libraries/tests') + .reply(500) + const ciVisibilityExporter = new CiVisibilityExporter({ port, isEarlyFlakeDetectionEnabled: true }) + + ciVisibilityExporter._resolveCanUseCiVisProtocol(true) + ciVisibilityExporter._libraryConfig = { isEarlyFlakeDetectionEnabled: true } + ciVisibilityExporter.getKnownTests({}, (err) => { + expect(err).not.to.be.null + expect(scope.isDone()).to.be.true + done() + }) + }) + it('should accept gzip if the exporter is gzip compatible', (done) => { + let requestHeaders = {} + const scope = nock(`http://localhost:${port}`) + .post('/api/v2/ci/libraries/tests') + .reply(200, function () { + requestHeaders = this.req.headers + + return zlib.gzipSync(JSON.stringify({ + data: { attributes: { test_full_names: ['suite1.test1', 'suite2.test2'] } } + })) + }, { + 'content-encoding': 'gzip' + }) + + const ciVisibilityExporter = new CiVisibilityExporter({ port, isEarlyFlakeDetectionEnabled: true }) + + ciVisibilityExporter._resolveCanUseCiVisProtocol(true) + ciVisibilityExporter._libraryConfig = { isEarlyFlakeDetectionEnabled: true } + ciVisibilityExporter._isGzipCompatible = true + ciVisibilityExporter.getKnownTests({}, (err, knownTests) => { + expect(err).to.be.null + expect(knownTests).to.eql(['suite1.test1', 'suite2.test2']) + expect(scope.isDone()).to.be.true + expect(requestHeaders['accept-encoding']).to.equal('gzip') + done() + }) + }) + it('should not accept gzip if the exporter is gzip incompatible', (done) => { + let requestHeaders = {} + const scope = nock(`http://localhost:${port}`) + .post('/api/v2/ci/libraries/tests') + .reply(200, function () { + requestHeaders = this.req.headers + + return JSON.stringify({ + data: { attributes: { test_full_names: ['suite1.test1', 'suite2.test2'] } } + }) + }) + + const ciVisibilityExporter = new CiVisibilityExporter({ port, isEarlyFlakeDetectionEnabled: true }) + + ciVisibilityExporter._resolveCanUseCiVisProtocol(true) + ciVisibilityExporter._libraryConfig = { isEarlyFlakeDetectionEnabled: true } + + ciVisibilityExporter._isGzipCompatible = false + + ciVisibilityExporter.getKnownTests({}, (err, knownTests) => { + expect(err).to.be.null + expect(knownTests).to.eql(['suite1.test1', 'suite2.test2']) + expect(scope.isDone()).to.be.true + expect(requestHeaders['accept-encoding']).not.to.equal('gzip') + done() + }) + }) + }) + }) }) diff --git a/packages/dd-trace/test/config.spec.js b/packages/dd-trace/test/config.spec.js index 55335b78d37..367749f08d6 100644 --- a/packages/dd-trace/test/config.spec.js +++ b/packages/dd-trace/test/config.spec.js @@ -1266,6 +1266,7 @@ describe('Config', () => { delete process.env.DD_CIVISIBILITY_ITR_ENABLED delete process.env.DD_CIVISIBILITY_GIT_UPLOAD_ENABLED delete process.env.DD_CIVISIBILITY_MANUAL_API_ENABLED + delete process.env.DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED options = {} }) context('ci visibility mode is enabled', () => { @@ -1312,6 +1313,15 @@ describe('Config', () => { const config = new Config(options) expect(config).to.nested.property('telemetry.enabled', true) }) + it('should enable early flake detection by default', () => { + const config = new Config(options) + expect(config).to.have.property('isEarlyFlakeDetectionEnabled', true) + }) + it('should disable early flake detection if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', () => { + process.env.DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED = 'false' + const config = new Config(options) + expect(config).to.have.property('isEarlyFlakeDetectionEnabled', false) + }) }) context('ci visibility mode is not enabled', () => { it('should not activate intelligent test runner or git metadata upload', () => {