diff --git a/src/commands/sarif/README.md b/src/commands/sarif/README.md index f55ce6c7b..65c4c6280 100644 --- a/src/commands/sarif/README.md +++ b/src/commands/sarif/README.md @@ -25,6 +25,8 @@ The positional arguments are the directories or file paths in which the SARIF re - `--max-concurrency` (default: `20`): number of concurrent uploads to the API. - `--dry-run` (default: `false`): runs the command without the final upload step. All other checks are performed. - `--no-verify` (default: `false`): runs the command without performing report validation on the CLI. +- `--no-ci-tags` (default: `false`): ignore the automatic detection of continuous integration environment variables. +- `--git-repository` (default: `current working directory`): reports git environment context from the specified repository. ### Environment variables @@ -34,9 +36,13 @@ Additionally, you may configure the `sarif` command with environment variables: - `DD_TAGS`: Set global tags applied to all spans. The format must be `key1:value1,key2:value2`. The upload process merges the tags passed on the command line with the tags in the `--tags` parameter. If a key appears in both `--tags` and `DD_TAGS`, the value in `DD_TAGS` takes precedence. - `DATADOG_SITE` or `DD_SITE`: choose your Datadog site, for example, datadoghq.com or datadoghq.eu. -### Optional dependencies +### Git context resolution -- [`git`](https://git-scm.com/downloads) is used for extracting repository metadata. +The Git context is resolved in the following order of priority: +1. Current process location +2. CI environment variables (can be disabled with: `--no-ci-tags` option) +3. Explicitly provided Git repository (through `--git-repository` option) +4. Override environment variables (`DD_GIT_*` variables) ### End-to-end testing process diff --git a/src/commands/sarif/__tests__/fixtures/gitconfig b/src/commands/sarif/__tests__/fixtures/gitconfig new file mode 100644 index 000000000..8768eaf91 --- /dev/null +++ b/src/commands/sarif/__tests__/fixtures/gitconfig @@ -0,0 +1,26 @@ +[core] + repositoryformatversion = 1 + filemode = false + bare = false + logallrefupdates = false + ignorecase = true + +[init] + defaultBranch = mock-branch + +[user] + name = MockUser123 + email = mock@fake.local + +[remote "mock-origin"] + url = https://mock-repo.local/fake.git + fetch = +refs/mocks/*:refs/remotes/mock-origin/* + +[branch "mock-branch"] + remote = mock-origin + merge = refs/heads/mock-branch + rebase = never # Unusual setting + +[hooks] + pre-commit = echo 'Mock pre-commit hook executed' + pre-push = echo 'Mock pre-push hook executed' diff --git a/src/commands/sarif/__tests__/upload.test.ts b/src/commands/sarif/__tests__/upload.test.ts index 251f8f680..6c90cfdec 100644 --- a/src/commands/sarif/__tests__/upload.test.ts +++ b/src/commands/sarif/__tests__/upload.test.ts @@ -1,6 +1,9 @@ +import fs from 'fs' import os from 'os' +import path from 'path' import {Cli} from 'clipanion/lib/advanced' +import simpleGit from 'simple-git' import {renderInvalidFile} from '../renderer' import {UploadSarifReportCommand} from '../upload' @@ -166,11 +169,11 @@ describe('upload', () => { }) describe('execute', () => { - const runCLI = async (paths: string[]) => { + const runCLI = async (args: string[]) => { const cli = makeCli() const context = createMockContext() as any process.env = {DATADOG_API_KEY: 'PLACEHOLDER'} - const code = await cli.run(['sarif', 'upload', '--env', 'ci', '--dry-run', ...paths], context) + const code = await cli.run(['sarif', 'upload', '--env', 'ci', '--dry-run', ...args], context) return {context, code} } @@ -209,17 +212,72 @@ describe('execute', () => { basePaths: [`${process.cwd()}/src/commands/sarif/__tests__/fixtures/subfolder`], concurrency: 20, env: 'ci', + spanTags: { + 'git.repository_url': 'DataDog/datadog-ci', + env: 'ci', + }, }) }) + test('absolute path when passing git repository', async () => { + const tmpdir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitPath-')) + try { + // Configure local git repository + const git = simpleGit(tmpdir) + setupLocalGitConfig(tmpdir) + + await git.init() + + // eslint-disable-next-line no-null/no-null + await git.commit('Initial commit', [], {'--allow-empty': null}) + const repositoryParam = `--git-repository=${tmpdir}` + + const {context, code} = await runCLI([ + repositoryParam, + process.cwd() + '/src/commands/sarif/__tests__/fixtures/subfolder', + ]) + + const output = context.stdout.toString().split(os.EOL) + expect(code).toBe(0) + + checkConsoleOutput(output, { + basePaths: [`${process.cwd()}/src/commands/sarif/__tests__/fixtures/subfolder`], + concurrency: 20, + env: 'ci', + spanTags: { + 'git.repository_url': 'mock-repo.local/fake.git', + 'git.branch': 'mock-branch', + 'git.commit.message': 'Initial commit', + 'git.commit.committer.email': 'mock@fake.local', + 'git.commit.committer.name': 'MockUser123', + 'git.commit.author.email': 'mock@fake.local', + 'git.commit.author.name': 'MockUser123', + env: 'ci', + }, + }) + } finally { + // Removed temporary git file + fs.rmSync(tmpdir, {recursive: true, force: true}) + } + }) + + test('absolute path when passing git repository which does not exist', async () => { + const nonExistingGitRepository = '/you/cannot/find/me' + const repositoryParam = `--git-repository=${nonExistingGitRepository}` + + // Pass a git repository which does not exist, command should fail + const {code} = await runCLI([repositoryParam, process.cwd() + '/src/commands/sarif/__tests__/fixtures/subfolder']) + expect(code).toBe(1) + }) + test('single file', async () => { const {context, code} = await runCLI([process.cwd() + '/src/commands/sarif/__tests__/fixtures/valid-results.sarif']) const output = context.stdout.toString().split(os.EOL) - const path = `${process.cwd()}/src/commands/sarif/__tests__/fixtures/valid-results.sarif` + const location = `${process.cwd()}/src/commands/sarif/__tests__/fixtures/valid-results.sarif` expect(code).toBe(0) expect(output[0]).toContain('DRY-RUN MODE ENABLED. WILL NOT UPLOAD SARIF REPORT') expect(output[1]).toContain('Starting upload with concurrency 20.') - expect(output[2]).toContain(`Will upload SARIF report file ${path}`) + expect(output[2]).toContain(`Will upload SARIF report file ${location}`) expect(output[3]).toContain('Only one upload per commit, env and tool') expect(output[4]).toContain(`Preparing upload for`) expect(output[4]).toContain(`env:ci`) @@ -228,9 +286,9 @@ describe('execute', () => { test('not found file', async () => { const {context, code} = await runCLI([process.cwd() + '/src/commands/sarif/__tests__/fixtures/not-found.sarif']) const output = context.stdout.toString().split(os.EOL) - const path = `${process.cwd()}/src/commands/sarif/__tests__/fixtures/not-found.sarif` + const location = `${process.cwd()}/src/commands/sarif/__tests__/fixtures/not-found.sarif` expect(code).toBe(1) - expect(output[0]).toContain(`Cannot find valid SARIF report files to upload in ${path}`) + expect(output[0]).toContain(`Cannot find valid SARIF report files to upload in ${location}`) expect(output[1]).toContain('Check the files exist and are valid.') }) }) @@ -239,6 +297,7 @@ interface ExpectedOutput { basePaths: string[] concurrency: number env: string + spanTags?: Record } const checkConsoleOutput = (output: string[], expected: ExpectedOutput) => { @@ -248,4 +307,31 @@ const checkConsoleOutput = (output: string[], expected: ExpectedOutput) => { expect(output[3]).toContain('Only one upload per commit, env and tool') expect(output[4]).toContain(`Preparing upload for`) expect(output[4]).toContain(`env:${expected.env}`) + + if (expected.spanTags) { + const regex = /with tags (\{.*\})/ + const match = output[5].match(regex) + expect(match).not.toBeNull() + + const spanTags = JSON.parse(match![1]) + Object.keys(expected.spanTags).forEach((k) => { + expect(spanTags[k]).not.toBeNull() + expect(spanTags[k]).toContain(expected.spanTags![k]) + }) + } +} + +const getFixtures = (file: string) => { + return path.join('./src/commands/sarif/__tests__/fixtures', file) +} + +const setupLocalGitConfig = (dir: string) => { + const gitDir = path.join(dir, '.git') + if (!fs.existsSync(gitDir)) { + fs.mkdirSync(gitDir, {recursive: true}) + } + + const configFixture = fs.readFileSync(getFixtures('gitconfig'), 'utf8') + const configPath = path.join(gitDir, '/config') + fs.writeFileSync(configPath, configFixture) } diff --git a/src/commands/sarif/renderer.ts b/src/commands/sarif/renderer.ts index 4cdb033c2..3ccd5e612 100644 --- a/src/commands/sarif/renderer.ts +++ b/src/commands/sarif/renderer.ts @@ -72,9 +72,11 @@ export const renderSuccessfulCommand = (fileCount: number, duration: number) => return fullStr } -export const renderDryRunUpload = (payload: Payload): string => `[DRYRUN] ${renderUpload(payload)}` +export const renderDryRunUpload = (payload: Payload): string => `[DRYRUN] ${renderUploadWithSpan(payload)}` export const renderUpload = (payload: Payload): string => `Uploading SARIF report in ${payload.reportPath}\n` +export const renderUploadWithSpan = (payload: Payload): string => + `Uploading SARIF report to ${payload.reportPath} with tags ${JSON.stringify(payload.spanTags)}\n` export const renderCommandInfo = ( basePaths: string[], diff --git a/src/commands/sarif/upload.ts b/src/commands/sarif/upload.ts index 7486d8418..cb3b600e8 100644 --- a/src/commands/sarif/upload.ts +++ b/src/commands/sarif/upload.ts @@ -64,6 +64,7 @@ export class UploadSarifReportCommand extends Command { private maxConcurrency = Option.String('--max-concurrency', '20', {validator: validation.isInteger()}) private serviceFromCli = Option.String('--service') private tags = Option.Array('--tags') + private gitPath = Option.String('--git-repository') private noVerify = Option.Boolean('--no-verify', false) private noCiTags = Option.Boolean('--no-ci-tags', false) @@ -110,7 +111,7 @@ export class UploadSarifReportCommand extends Command { // Always using the posix version to avoid \ on Windows. this.basePaths = this.basePaths.map((basePath) => path.posix.normalize(basePath)) - const spanTags = await getSpanTags(this.config, this.tags, !this.noCiTags) + const spanTags = await getSpanTags(this.config, this.tags, !this.noCiTags, this.gitPath) // Gather any missing mandatory git fields to display to the user const missingGitFields = getMissingRequiredGitTags(spanTags) @@ -133,7 +134,7 @@ export class UploadSarifReportCommand extends Command { this.context.stdout.write( renderCommandInfo(this.basePaths, env, sha, this.maxConcurrency, this.dryRun, this.noVerify) ) - const upload = (p: Payload) => this.uploadSarifReport(api, p) + const upload = (payload: Payload) => this.uploadSarifReport(api, payload) const initialTime = new Date().getTime() diff --git a/src/commands/sbom/README.md b/src/commands/sbom/README.md index 75aaab6b8..42a74586d 100644 --- a/src/commands/sbom/README.md +++ b/src/commands/sbom/README.md @@ -7,16 +7,20 @@ This command lets you upload SBOM files to the Datadog intake endpoint. - CycloneDX 1.4 - CycloneDX 1.5 + - CycloneDX 1.6 ## Usage ```bash -datadog-ci sbom upload +datadog-ci sbom upload [--env] [--no-ci-tags] [--git-repository] [--debug] ``` ### Optional arguments -- `--env` is a string that represents the environment in which you want your tests to appear. +- `--env` (default: `ci`): represents the environment in which you want your sbom to appear. +- `--no-ci-tags` (default: `false`): ignore the automatic detection of continuous integration environment variables. +- `--git-repository` (default: `current working directory`): reports git environment context from the specified repository. +- `--debug` (default: `false`): output debug logs. ### Environment variables @@ -26,6 +30,14 @@ The following environment variables must be defined: - `DD_APP_KEY`: the App key to use - `DD_API_KEY`: the API key to use +### Git context resolution + +The Git context is resolved in the following order of priority: +1. Current process location +2. CI environment variables (can be disabled with: `--no-ci-tags` option) +3. Explicitly provided Git repository (through --git-repository option) +4. Override environment variables (`DD_GIT_*` variables) + ## Development When developing software, you can try with the following command: diff --git a/src/commands/sbom/__tests__/fixtures/gitconfig b/src/commands/sbom/__tests__/fixtures/gitconfig new file mode 100644 index 000000000..8768eaf91 --- /dev/null +++ b/src/commands/sbom/__tests__/fixtures/gitconfig @@ -0,0 +1,26 @@ +[core] + repositoryformatversion = 1 + filemode = false + bare = false + logallrefupdates = false + ignorecase = true + +[init] + defaultBranch = mock-branch + +[user] + name = MockUser123 + email = mock@fake.local + +[remote "mock-origin"] + url = https://mock-repo.local/fake.git + fetch = +refs/mocks/*:refs/remotes/mock-origin/* + +[branch "mock-branch"] + remote = mock-origin + merge = refs/heads/mock-branch + rebase = never # Unusual setting + +[hooks] + pre-commit = echo 'Mock pre-commit hook executed' + pre-push = echo 'Mock pre-push hook executed' diff --git a/src/commands/sbom/__tests__/payload.test.ts b/src/commands/sbom/__tests__/payload.test.ts index 0bfabdcbe..27c917b47 100644 --- a/src/commands/sbom/__tests__/payload.test.ts +++ b/src/commands/sbom/__tests__/payload.test.ts @@ -1,7 +1,11 @@ import fs from 'fs' +import os from 'os' +import path from 'path' + +import simpleGit from 'simple-git' import {DatadogCiConfig} from '../../../helpers/config' -import {getSpanTags} from '../../../helpers/tags' +import {getSpanTags, getMissingRequiredGitTags} from '../../../helpers/tags' import {generatePayload} from '../payload' import {DependencyLanguage, Location} from '../types' @@ -343,4 +347,78 @@ describe('generation of payload', () => { expect(dependencies[1].name).toEqual('jinja2') expect(dependencies[1].version).toEqual('3.1.5') }) + + test('should correctly work with a CycloneDX 1.4 file and passing git repository', async () => { + const tmpdir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitPath-')) + try { + // Configure local git repository + const git = simpleGit(tmpdir) + setupLocalGitConfig(tmpdir) + await git.init() + // eslint-disable-next-line no-null/no-null + await git.commit('Initial commit', [], {'--allow-empty': null}) + + const sbomFile = './src/commands/sbom/__tests__/fixtures/sbom.1.4.ok.json' + const sbomContent = JSON.parse(fs.readFileSync(sbomFile).toString('utf8')) + const config: DatadogCiConfig = { + apiKey: undefined, + env: undefined, + envVarTags: undefined, + } + + // Pass git directory to load git context + const tags = await getSpanTags(config, [], true, tmpdir) + expect(getMissingRequiredGitTags(tags)).toHaveLength(0) + + const payload = generatePayload(sbomContent, tags, 'service', 'env') + expect(payload).not.toBeNull() + expect(payload?.id).toStrictEqual(expect.any(String)) + + // Local git repository should be reported + expect(payload?.commit.sha).toStrictEqual(expect.any(String)) + expect(payload?.commit.author_name).toStrictEqual('MockUser123') + expect(payload?.commit.author_email).toStrictEqual('mock@fake.local') + expect(payload?.commit.committer_name).toStrictEqual('MockUser123') + expect(payload?.commit.committer_email).toStrictEqual('mock@fake.local') + expect(payload?.commit.branch).toStrictEqual('mock-branch') + expect(payload?.repository.url).toContain('https://mock-repo.local/fake.git') + expect(payload?.dependencies.length).toBe(62) + expect(payload?.dependencies[0].name).toBe('stack-cors') + expect(payload?.dependencies[0].version).toBe('1.3.0') + expect(payload?.dependencies[0].licenses.length).toBe(0) + expect(payload?.dependencies[0].language).toBe(DependencyLanguage.PHP) + } finally { + // Removed temporary git file + fs.rmSync(tmpdir, {recursive: true, force: true}) + } + }) + + test('should fail to read git information', async () => { + const nonExistingGitRepository = '/you/cannot/find/me' + const config: DatadogCiConfig = { + apiKey: undefined, + env: undefined, + envVarTags: undefined, + } + + // Pass non existing git directory to load git context + // It is missing all git tags. + const tags = await getSpanTags(config, [], true, nonExistingGitRepository) + expect(getMissingRequiredGitTags(tags).length).toBeGreaterThanOrEqual(1) + }) }) + +const getFixtures = (file: string) => { + return path.join('./src/commands/sbom/__tests__/fixtures', file) +} + +const setupLocalGitConfig = (dir: string) => { + const gitDir = path.join(dir, '.git') + if (!fs.existsSync(gitDir)) { + fs.mkdirSync(gitDir, {recursive: true}) + } + + const configFixture = fs.readFileSync(getFixtures('gitconfig'), 'utf8') + const configPath = path.join(gitDir, '/config') + fs.writeFileSync(configPath, configFixture) +} diff --git a/src/commands/sbom/upload.ts b/src/commands/sbom/upload.ts index 048a1bb3e..baf70b7dc 100644 --- a/src/commands/sbom/upload.ts +++ b/src/commands/sbom/upload.ts @@ -48,6 +48,7 @@ export class UploadSbomCommand extends Command { private serviceFromCli = Option.String('--service') private env = Option.String('--env', 'ci') private tags = Option.Array('--tags') + private gitPath = Option.String('--git-repository') private debug = Option.Boolean('--debug') private noCiTags = Option.Boolean('--no-ci-tags', false) @@ -114,7 +115,7 @@ export class UploadSbomCommand extends Command { this.config.appKey ) - const tags = await getSpanTags(this.config, this.tags, !this.noCiTags) + const tags = await getSpanTags(this.config, this.tags, !this.noCiTags, this.gitPath) // Gather any missing mandatory git fields to display to the user const missingGitFields = getMissingRequiredGitTags(tags) @@ -135,7 +136,7 @@ export class UploadSbomCommand extends Command { if (!validateSbomFileAgainstSchema(basePath, validator, !!this.debug)) { this.context.stdout.write( - 'SBOM file not fully compliant against CycloneDX 1.4 or 1.5 specifications (use --debug to get validation error)\n' + 'SBOM file not fully compliant against CycloneDX 1.4, 1.5 or 1.6 specifications (use --debug to get validation error)\n' ) } if (!validateFileAgainstToolRequirements(basePath, !!this.debug)) { diff --git a/src/helpers/__tests__/tags.test.ts b/src/helpers/__tests__/tags.test.ts index a2771a11e..447d0dcc7 100644 --- a/src/helpers/__tests__/tags.test.ts +++ b/src/helpers/__tests__/tags.test.ts @@ -1,6 +1,7 @@ import {BaseContext} from 'clipanion' import simpleGit from 'simple-git' +import {DatadogCiConfig} from '../config' import {SpanTags} from '../interfaces' import { parseTags, @@ -153,6 +154,15 @@ describe('parseMetricsFile', () => { }) describe('getSpanTags', () => { + beforeEach(() => { + // CI defined by default env vars - clearing them is needed for tests to pass in the CI + process.env.GITHUB_ACTIONS = '' + process.env.GITHUB_REPOSITORY = '' + process.env.GITHUB_HEAD_REF = '' + process.env.GITHUB_SHA = '' + process.env.GITHUB_RUN_ID = '' + }) + test('should parse DD_TAGS and DD_ENV environment variables', async () => { process.env.DD_TAGS = 'key1:https://google.com,key2:value2' process.env.DD_ENV = 'ci' @@ -187,6 +197,65 @@ describe('getSpanTags', () => { key2: 'value2', }) }) + test('should prioritized git context accordingly', async () => { + const config: DatadogCiConfig = { + apiKey: undefined, + env: undefined, + envVarTags: undefined, + } + + // Define CI env variables + process.env.GITHUB_ACTIONS = 'true' + process.env.GITHUB_REPOSITORY = 'git@github.com:DataDog/datadog-ci' + process.env.GITHUB_HEAD_REF = 'prod' + process.env.GITHUB_SHA = 'commitSHA' + process.env.GITHUB_RUN_ID = '17' + + // Mock simpleGit (used to read git context) + ;(simpleGit as jest.Mock).mockImplementation(() => ({ + branch: () => ({current: 'main'}), + listRemote: async (git: any): Promise => 'https://www.github.com/datadog/safe-repository', + revparse: () => 'mockSHA', + show: (input: string[]) => { + if (input[1] === '--format=%s') { + return 'commit message' + } + + return 'authorName,authorEmail,authorDate,committerName,committerEmail,committerDate' + }, + })) + + // By default, the 'git' tags reported are the CI env variables + let spanTags: SpanTags = await getSpanTags(config, [], true) + expect(spanTags['ci.pipeline.id']).toEqual('17') + expect(spanTags['git.repository_url']).toContain('DataDog/datadog-ci') + expect(spanTags['git.branch']).toEqual('prod') + expect(spanTags['git.commit.sha']).toEqual('commitSHA') + + // re-query span tags while specifying a git directory: + // if should prefer the git directory over env variables. + spanTags = await getSpanTags(config, [], true, 'path/to/mocked/SimpleGit') + expect(spanTags).toMatchObject({ + 'ci.pipeline.id': '17', + 'git.repository_url': 'https://www.github.com/datadog/safe-repository', + 'git.branch': 'main', + 'git.commit.sha': 'mockSHA', + }) + + // Configuring DD_GIT* overrides + process.env.DD_GIT_REPOSITORY_URL = 'https://www.github.com/datadog/not-so-safe-repository' + process.env.DD_GIT_BRANCH = 'staging' + process.env.DD_GIT_COMMIT_SHA = 'override' + + // git tags are coming from overrides + spanTags = await getSpanTags(config, [], true, 'path/to/mockedSimpleGit') + expect(spanTags).toMatchObject({ + 'ci.pipeline.id': '17', + 'git.repository_url': 'https://www.github.com/datadog/not-so-safe-repository', + 'git.branch': 'staging', + 'git.commit.sha': 'override', + }) + }) }) describe('sarif and sbom upload required git tags', () => { diff --git a/src/helpers/git/format-git-span-data.ts b/src/helpers/git/format-git-span-data.ts index a8bfb7251..a44667881 100644 --- a/src/helpers/git/format-git-span-data.ts +++ b/src/helpers/git/format-git-span-data.ts @@ -18,10 +18,10 @@ import {filterSensitiveInfoFromRepository} from '../utils' import {gitAuthorAndCommitter, gitBranch, gitHash, gitMessage, gitRepositoryURL} from './get-git-data' -export const getGitMetadata = async (): Promise => { +export const getGitMetadata = async (repositoryPath?: string): Promise => { try { const git = simpleGit({ - baseDir: process.cwd(), + baseDir: repositoryPath || process.cwd(), binary: 'git', // We are invoking at most 5 git commands at the same time. maxConcurrentProcesses: 5, diff --git a/src/helpers/tags.ts b/src/helpers/tags.ts index 30ef5f1f7..9ff8e8f74 100644 --- a/src/helpers/tags.ts +++ b/src/helpers/tags.ts @@ -236,18 +236,19 @@ export const getMissingRequiredGitTags = (tags: SpanTags): string[] => { export const getSpanTags = async ( config: DatadogCiConfig, additionalTags: string[] | undefined, - includeCiTags: boolean + includeCiTags: boolean, + gitPath?: string ): Promise => { const ciSpanTags = includeCiTags ? getCISpanTags() : [] - const gitSpanTags = await getGitMetadata() + const gitSpanTags = await getGitMetadata(gitPath) const userGitSpanTags = getUserGitSpanTags() const envVarTags = config.envVarTags ? parseTags(config.envVarTags.split(',')) : {} const cliTags = additionalTags ? parseTags(additionalTags) : {} return { - ...gitSpanTags, - ...ciSpanTags, + // if users specify a git path to read from, we prefer git env variables over the CI context one + ...(gitPath ? {...ciSpanTags, ...gitSpanTags} : {...gitSpanTags, ...ciSpanTags}), ...userGitSpanTags, // User-provided git tags have precedence over the ones we get from the git command ...cliTags, ...envVarTags,