-
-
Notifications
You must be signed in to change notification settings - Fork 1.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
ci: Only run E2E test apps that are affected on PRs (#14254)
This PR aims to solve two things: 1. Do not require us to keep a list of test-applications to run on CI. This is prone to be forgotten, and then certain tests do not run on CI and we may not notice (this has happened more than once in the past 😬 ) 2. Do not run E2E test applications that are unrelated to anything that was changed. With this PR, instead of keeping a list of E2E test jobs we want to run manually in the build.yml file, this is now inferred (on CI) by a script based on the nx dependency graph and some config in the test apps package.json. This is how it works: 1. Pick all folders in test-applications, by default all of them should run. --> This will "fix" the problem we had sometimes that we forgot to add test apps to the build.yml so they don't actually run on CI :grimacing: 3. For each test app, look at it's dependencies (and devDependencies), and see if any of them has changed in the current PR (based on nx affected projects). So e.g. if you change something in browser, only E2E test apps that have any dependency that uses browser will run, so e.g. node-express will be skipped. 4. Additionally, you can configure variants in the test apps package.json - instead of defining this in the workflow file, it has to be defined in the test app itself now. 5. Finally, a test app can be marked as optional in the package.json, and can also have optional variants to run. 6. The E2E test builds the required & optional matrix on the fly based on these things. 7. In pushes (=on develop/release branches) we just run everything. 8. If anything inside of the e2e-tests package changed, run everything. --> This could be further optimized to only run changed test apps, but this was the easy solution for now. ## Example test runs * In a PR with no changes related to the E2E tests, nothing is run: https://github.com/getsentry/sentry-javascript/actions/runs/11838604965 * In a CI run for a push, all E2E tests are run: https://github.com/getsentry/sentry-javascript/actions/runs/11835834748 * In a PR with a change in e2e-tests itself, all E2E tests are run: https://github.com/getsentry/sentry-javascript/actions/runs/11836668035 * In a PR with a change in the node package, only related E2E tests are run: https://github.com/getsentry/sentry-javascript/actions/runs/11838274483
- Loading branch information
1 parent
be893c0
commit 6b2c304
Showing
21 changed files
with
326 additions
and
192 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,155 @@ | ||
| import { execSync } from 'child_process'; | ||
| import * as fs from 'fs'; | ||
| import * as path from 'path'; | ||
| import { dirname } from 'path'; | ||
| import { parseArgs } from 'util'; | ||
| import { sync as globSync } from 'glob'; | ||
|
|
||
| interface MatrixInclude { | ||
| /** The test application (directory) name. */ | ||
| 'test-application': string; | ||
| /** Optional override for the build command to run. */ | ||
| 'build-command'?: string; | ||
| /** Optional override for the assert command to run. */ | ||
| 'assert-command'?: string; | ||
| /** Optional label for the test run. If not set, defaults to value of `test-application`. */ | ||
| label?: string; | ||
| } | ||
|
|
||
| interface PackageJsonSentryTestConfig { | ||
| /** If this is true, the test app is optional. */ | ||
| optional?: boolean; | ||
| /** Variant configs that should be run in non-optional test runs. */ | ||
| variants?: Partial<MatrixInclude>[]; | ||
| /** Variant configs that should be run in optional test runs. */ | ||
| optionalVariants?: Partial<MatrixInclude>[]; | ||
| /** Skip this test app for matrix generation. */ | ||
| skip?: boolean; | ||
| } | ||
|
|
||
| /** | ||
| * This methods generates a matrix for the GitHub Actions workflow to run the E2E tests. | ||
| * It checks which test applications are affected by the current changes in the PR and then generates a matrix | ||
| * including all test apps that have at least one dependency that was changed in the PR. | ||
| * If no `--base=xxx` is provided, it will output all test applications. | ||
| * | ||
| * If `--optional=true` is set, it will generate a matrix of optional test applications only. | ||
| * Otherwise, these will be skipped. | ||
| */ | ||
| function run(): void { | ||
| const { values } = parseArgs({ | ||
| args: process.argv.slice(2), | ||
| options: { | ||
| base: { type: 'string' }, | ||
| head: { type: 'string' }, | ||
| optional: { type: 'string', default: 'false' }, | ||
| }, | ||
| }); | ||
|
|
||
| const { base, head, optional } = values; | ||
|
|
||
| const testApplications = globSync('*/package.json', { | ||
| cwd: `${__dirname}/../test-applications`, | ||
| }).map(filePath => dirname(filePath)); | ||
|
|
||
| // If `--base=xxx` is defined, we only want to get test applications changed since that base | ||
| // Else, we take all test applications (e.g. on push) | ||
| const includedTestApplications = base | ||
| ? getAffectedTestApplications(testApplications, { base, head }) | ||
| : testApplications; | ||
|
|
||
| const optionalMode = optional === 'true'; | ||
| const includes: MatrixInclude[] = []; | ||
|
|
||
| includedTestApplications.forEach(testApp => { | ||
| addIncludesForTestApp(testApp, includes, { optionalMode }); | ||
| }); | ||
|
|
||
| // We print this to the output, so the GHA can use it for the matrix | ||
| // eslint-disable-next-line no-console | ||
| console.log(`matrix=${JSON.stringify({ include: includes })}`); | ||
| } | ||
|
|
||
| function addIncludesForTestApp( | ||
| testApp: string, | ||
| includes: MatrixInclude[], | ||
| { optionalMode }: { optionalMode: boolean }, | ||
| ): void { | ||
| const packageJson = getPackageJson(testApp); | ||
|
|
||
| const shouldSkip = packageJson.sentryTest?.skip || false; | ||
| const isOptional = packageJson.sentryTest?.optional || false; | ||
| const variants = (optionalMode ? packageJson.sentryTest?.optionalVariants : packageJson.sentryTest?.variants) || []; | ||
|
|
||
| if (shouldSkip) { | ||
| return; | ||
| } | ||
|
|
||
| // Add the basic test-application itself, if it is in the current mode | ||
| if (optionalMode === isOptional) { | ||
| includes.push({ | ||
| 'test-application': testApp, | ||
| }); | ||
| } | ||
|
|
||
| variants.forEach(variant => { | ||
| includes.push({ | ||
| 'test-application': testApp, | ||
| ...variant, | ||
| }); | ||
| }); | ||
| } | ||
|
|
||
| function getSentryDependencies(appName: string): string[] { | ||
| const packageJson = getPackageJson(appName) || {}; | ||
|
|
||
| const dependencies = { | ||
| ...packageJson.devDependencies, | ||
| ...packageJson.dependencies, | ||
| }; | ||
|
|
||
| return Object.keys(dependencies).filter(key => key.startsWith('@sentry')); | ||
| } | ||
|
|
||
| function getPackageJson(appName: string): { | ||
| dependencies?: { [key: string]: string }; | ||
| devDependencies?: { [key: string]: string }; | ||
| sentryTest?: PackageJsonSentryTestConfig; | ||
| } { | ||
| const fullPath = path.resolve(__dirname, '..', 'test-applications', appName, 'package.json'); | ||
|
|
||
| if (!fs.existsSync(fullPath)) { | ||
| throw new Error(`Could not find package.json for ${appName}`); | ||
| } | ||
|
|
||
| return JSON.parse(fs.readFileSync(fullPath, 'utf8')); | ||
| } | ||
|
|
||
| run(); | ||
|
|
||
| function getAffectedTestApplications( | ||
| testApplications: string[], | ||
| { base = 'develop', head }: { base?: string; head?: string }, | ||
| ): string[] { | ||
| const additionalArgs = [`--base=${base}`]; | ||
|
|
||
| if (head) { | ||
| additionalArgs.push(`--head=${head}`); | ||
| } | ||
|
|
||
| const affectedProjects = execSync(`yarn --silent nx show projects --affected ${additionalArgs.join(' ')}`) | ||
| .toString() | ||
| .split('\n') | ||
| .map(line => line.trim()) | ||
| .filter(Boolean); | ||
|
|
||
| // If something in e2e tests themselves are changed, just run everything | ||
| if (affectedProjects.includes('@sentry-internal/e2e-tests')) { | ||
| return testApplications; | ||
| } | ||
|
|
||
| return testApplications.filter(testApp => { | ||
| const sentryDependencies = getSentryDependencies(testApp); | ||
| return sentryDependencies.some(dep => affectedProjects.includes(dep)); | ||
| }); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -26,5 +26,8 @@ | |
| }, | ||
| "volta": { | ||
| "extends": "../../package.json" | ||
| }, | ||
| "sentryTest": { | ||
| "optional": true | ||
| } | ||
| } | ||
Oops, something went wrong.