From f229c24857c3ddc4fe66d628d174de100fb97ad4 Mon Sep 17 00:00:00 2001 From: Tre' Seymour Date: Wed, 1 Apr 2020 14:23:47 -0600 Subject: [PATCH] Add the vcs info --- .ci/Jenkinsfile_coverage | 88 +++++++++++-------- .../code_coverage/ingest_coverage/index.js | 6 +- .../code_coverage/ingest_coverage/process.js | 42 ++++++--- .../ingest_coverage/transforms.js | 8 ++ 4 files changed, 93 insertions(+), 51 deletions(-) diff --git a/.ci/Jenkinsfile_coverage b/.ci/Jenkinsfile_coverage index a24480277a0e1..8fd519d9781ae 100644 --- a/.ci/Jenkinsfile_coverage +++ b/.ci/Jenkinsfile_coverage @@ -9,85 +9,96 @@ kibanaPipeline(timeoutMinutes: 180) { 'CODE_COVERAGE=1', // Needed for multiple ci scripts, such as remote.ts, test/scripts/*.sh, schema.js, etc. ]) { -// workers.base(name: 'fail-fast-worker', label: 'tests-l', ramDisk: false, bootstrapped: false) { -// maybeFailFast('secret/kibana-issues/prod/coverage/elasticsearch') -// } + workers.base(name: 'fail-fast-worker', label: 'tests-l', ramDisk: false, bootstrapped: false) { + maybeFailFast('secret/kibana-issues/prod/coverage/elasticsearch') + } } -// parallel([ -// 'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'), -// 'x-pack-intake-agent': { -// withEnv([ -// 'NODE_ENV=test' // Needed for jest tests only -// ]) { -// workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh')() -// } -// }, -// 'kibana-oss-agent' : workers.functional( -// 'kibana-oss-tests', -// { kibanaPipeline.buildOss() }, -// ossProks() -// ), -// 'kibana-xpack-agent' : workers.functional( -// 'kibana-xpack-tests', -// { kibanaPipeline.buildXpack() }, -// xpackProks() -// ), -// ]) + parallel([ + 'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'), + 'x-pack-intake-agent': { + withEnv([ + 'NODE_ENV=test' // Needed for jest tests only + ]) { + workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh')() + } + }, + 'kibana-oss-agent' : workers.functional( + 'kibana-oss-tests', + { kibanaPipeline.buildOss() }, + ossProks() + ), + 'kibana-xpack-agent' : workers.functional( + 'kibana-xpack-tests', + { kibanaPipeline.buildXpack() }, + xpackProks() + ), + ]) workers.base(name: 'coverage-worker', label: 'tests-l', ramDisk: false, bootstrapped: false) { -// kibanaPipeline.downloadCoverageArtifacts() + kibanaPipeline.downloadCoverageArtifacts() def timestamp = new Date(currentBuild.startTimeInMillis).format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone("UTC")) withEnv([ "TIME_STAMP=${timestamp}", "QA_TEAM_ROCKS=true" ]) { - listAndCollectVcsEnvVars() + collectVcsInfo() def vaultSecret = 'secret/kibana-issues/prod/coverage/elasticsearch' withVaultSecret(secret: vaultSecret, secret_field: 'host', variable_name: 'HOST_FROM_VAULT') { withVaultSecret(secret: vaultSecret, secret_field: 'username', variable_name: 'USER_FROM_VAULT') { withVaultSecret(secret: vaultSecret, secret_field: 'password', variable_name: 'PASS_FROM_VAULT') { kibanaPipeline.bash(""" + source src/dev/ci_setup/setup_env.sh + + # bootstrap from x-pack folder + cd x-pack + yarn kbn bootstrap --prefer-offline + + # Return to project root + cd .. + . src/dev/code_coverage/shell_scripts/extract_archives.sh + . src/dev/code_coverage/shell_scripts/fix_html_reports_parallel.sh - echo "### Print vcs info file" - cat VCS_INFO.txt || echo "### VCS Info file NOT found" + . src/dev/code_coverage/shell_scripts/merge_jest_and_functional.sh + . src/dev/code_coverage/shell_scripts/copy_mocha_reports.sh + . src/dev/code_coverage/shell_scripts/ingest_coverage.sh ${BUILD_NUMBER} ${env.BUILD_URL} """, "### Bootstrap shell and kibana env, merge and ingest code coverage") } } } -// kibanaCoverage.uploadCoverageStaticSite(TIME_STAMP) + kibanaCoverage.uploadCoverageStaticSite(TIME_STAMP) } -// sh 'tar -czf kibana-jest-coverage.tar.gz target/kibana-coverage/jest-combined/*' -// kibanaPipeline.uploadCoverageArtifacts("coverage/jest-combined", 'kibana-jest-coverage.tar.gz') -// sh 'tar -czf kibana-functional-coverage.tar.gz target/kibana-coverage/functional-combined/*' -// kibanaPipeline.uploadCoverageArtifacts("coverage/functional-combined", 'kibana-functional-coverage.tar.gz') -// sh 'tar -czf kibana-mocha-coverage.tar.gz target/kibana-coverage/mocha-combined/*' -// kibanaPipeline.uploadCoverageArtifacts("coverage/mocha-combined", 'kibana-mocha-coverage.tar.gz') + sh 'tar -czf kibana-jest-coverage.tar.gz target/kibana-coverage/jest-combined/*' + kibanaPipeline.uploadCoverageArtifacts("coverage/jest-combined", 'kibana-jest-coverage.tar.gz') + sh 'tar -czf kibana-functional-coverage.tar.gz target/kibana-coverage/functional-combined/*' + kibanaPipeline.uploadCoverageArtifacts("coverage/functional-combined", 'kibana-functional-coverage.tar.gz') + sh 'tar -czf kibana-mocha-coverage.tar.gz target/kibana-coverage/mocha-combined/*' + kibanaPipeline.uploadCoverageArtifacts("coverage/mocha-combined", 'kibana-mocha-coverage.tar.gz') } } -// kibanaPipeline.sendMail() + kibanaPipeline.sendMail() } -def listAndCollectVcsEnvVars() { +def collectVcsInfo() { kibanaPipeline.bash( ''' predicate() { x=$1 if [ -n "$x" ]; then - echo "### Defined: ${x}" + return else echo "### 1 or more variables that Code Coverage needs, are undefined" exit 1 @@ -105,9 +116,8 @@ def listAndCollectVcsEnvVars() { for X in "${!XS[@]}"; do { - echo "### Checking: $X" predicate "${XS[X]}" - echo " ${XS[X]}" >> VCS_INFO.txt + echo "${XS[X]}" >> VCS_INFO.txt } done diff --git a/src/dev/code_coverage/ingest_coverage/index.js b/src/dev/code_coverage/ingest_coverage/index.js index eb4f34e1b1cb7..4fa9f28f3833d 100644 --- a/src/dev/code_coverage/ingest_coverage/index.js +++ b/src/dev/code_coverage/ingest_coverage/index.js @@ -35,8 +35,10 @@ export function runCoverageIngestionCli() { if (flags.path === '') throw createFlagError('please provide a single --path flag'); if (flags.verbose) log.verbose(`Verbose logging enabled`); - const jsonSummaryPath = resolve(ROOT, flags.path); - parseAndProcess({ jsonSummaryPath }, log); + const resolveRoot = resolve.bind(null, ROOT); + const jsonSummaryPath = resolveRoot(flags.path); + const vcsInfoFilePath = resolveRoot('VCS_INFO.txt'); + parseAndProcess({ jsonSummaryPath, vcsInfoFilePath }, log); }, { description: ` diff --git a/src/dev/code_coverage/ingest_coverage/process.js b/src/dev/code_coverage/ingest_coverage/process.js index 2a69c5296650f..be18a76555887 100644 --- a/src/dev/code_coverage/ingest_coverage/process.js +++ b/src/dev/code_coverage/ingest_coverage/process.js @@ -17,10 +17,10 @@ * under the License. */ -import { fromEventPattern, of } from 'rxjs'; -import { concatMap, delay, map } from 'rxjs/operators'; +import { fromEventPattern, of, fromEvent } from 'rxjs'; +import { concatMap, delay, map, takeUntil } from 'rxjs/operators'; import jsonStream from './json_stream'; -import { pipe, noop, green } from './utils'; +import { pipe, noop, green, always } from './utils'; import { ingest } from './ingest'; import { staticSite, @@ -31,8 +31,11 @@ import { buildId, coveredFilePath, ciRunUrl, + itemizeVcs, } from './transforms'; import { resolve } from 'path'; +import { createReadStream } from 'fs'; +import readline from 'readline'; const KIBANA_ROOT_PATH = '../../../..'; const KIBANA_ROOT = resolve(__dirname, KIBANA_ROOT_PATH); @@ -43,19 +46,15 @@ const addPrePopulatedTimeStamp = addTimeStamp(process.env.TIME_STAMP); const prokStatsTimeStampBuildId = pipe(statsAndstaticSiteUrl, buildId, addPrePopulatedTimeStamp); const addTestRunnerAndStaticSiteUrl = pipe(testRunner, staticSite(staticSiteUrlBase)); -export default ({ jsonSummaryPath }, log) => { - log.debug(`### Code coverage ingestion set to delay for: ${green(ms)} ms`); - log.debug(`### KIBANA_ROOT: \n\t${green(KIBANA_ROOT)}`); - log.debug(`### Ingesting from summary json: \n\t[${green(jsonSummaryPath)}]`); - - validateRoot(KIBANA_ROOT, log); - +const execute = jsonSummaryPath => log => vcsInfo => { const objStream = jsonStream(jsonSummaryPath).on('done', noop); + const itemizeVcsInfo = itemizeVcs(vcsInfo); fromEventPattern(_ => objStream.on('node', '!.*', _)) .pipe( map(prokStatsTimeStampBuildId), map(coveredFilePath), + map(itemizeVcsInfo), map(ciRunUrl), map(addJsonSummaryPath(jsonSummaryPath)), map(addTestRunnerAndStaticSiteUrl), @@ -64,6 +63,29 @@ export default ({ jsonSummaryPath }, log) => { .subscribe(ingest(log)); }; +export default ({ jsonSummaryPath, vcsInfoFilePath }, log) => { + log.debug(`### Code coverage ingestion set to delay for: ${green(ms)} ms`); + log.debug(`### KIBANA_ROOT: \n\t${green(KIBANA_ROOT)}`); + log.debug(`### Ingesting from summary json: \n\t[${green(jsonSummaryPath)}]`); + + validateRoot(KIBANA_ROOT, log); + + const vcsInfo = []; + const vcsInfoLines$ = vcsInfoFilePath => { + const rl = readline.createInterface({ input: createReadStream(vcsInfoFilePath) }); + return fromEvent(rl, 'line').pipe(takeUntil(fromEvent(rl, 'close'))); + }; + + const executeWithPath = execute(jsonSummaryPath)(log); + const mutate = x => vcsInfo.push(x.trimStart().trimEnd()); + + vcsInfoLines$(vcsInfoFilePath).subscribe( + mutate, + err => console.log('Error: %s', err), + always(executeWithPath(vcsInfo)) + ); +}; + function validateRoot(x, log) { return /kibana$/.test(x) ? noop() : log.warning(`!!! 'kibana' NOT FOUND in ROOT: ${x}\n`); } diff --git a/src/dev/code_coverage/ingest_coverage/transforms.js b/src/dev/code_coverage/ingest_coverage/transforms.js index 9a272809e495c..9a23492c3f699 100644 --- a/src/dev/code_coverage/ingest_coverage/transforms.js +++ b/src/dev/code_coverage/ingest_coverage/transforms.js @@ -106,6 +106,14 @@ export const ciRunUrl = obj => { }; }; +export const itemizeVcs = vcsInfo => obj => { + const [branch, sha, author, commitMsg] = vcsInfo; + return { + ...obj, + vcs: { branch, sha, author, commitMsg }, + }; +}; + export const testRunner = obj => { const { jsonSummaryPath } = obj;