From e8484a00912c80db400971f61626bd7526c90add Mon Sep 17 00:00:00 2001 From: Dan Muller Date: Tue, 17 Nov 2020 23:02:49 +0000 Subject: [PATCH] perf(cypress): pack cypress runfiles into a single tar --- packages/cypress/BUILD.bazel | 1 + packages/cypress/internal/install-cypress.js | 138 ++++++++---------- packages/cypress/internal/run-cypress.js | 84 +++++++++-- .../internal/template.cypress_web_test.bzl | 15 +- packages/cypress/package.json | 3 + packages/cypress/test/package.json | 5 +- packages/cypress/test/yarn.lock | 49 +++++++ 7 files changed, 196 insertions(+), 99 deletions(-) diff --git a/packages/cypress/BUILD.bazel b/packages/cypress/BUILD.bazel index a2e9fa164f..6e8170242f 100644 --- a/packages/cypress/BUILD.bazel +++ b/packages/cypress/BUILD.bazel @@ -95,6 +95,7 @@ pkg_npm( "@build_bazel_rules_nodejs//packages/cypress:internal/plugins/base.js": "TEMPLATED_node_modules_workspace_name//@bazel/cypress:internal/plugins/base.js", "@build_bazel_rules_nodejs//packages/cypress:internal/plugins/index.template.js": "TEMPLATED_node_modules_workspace_name//@bazel/cypress:internal/plugins/index.template.js", "@build_bazel_rules_nodejs//packages/cypress:internal/run-cypress.js": "TEMPLATED_node_modules_workspace_name//@bazel/cypress:internal/run-cypress.js", + "TEMPLATED_node_modules_workspace_name//tar": "TEMPLATED_node_modules_workspace_name//@bazel/cypress", }, deps = [ ":npm_version_check", diff --git a/packages/cypress/internal/install-cypress.js b/packages/cypress/internal/install-cypress.js index e8267088ee..7ff7feeb46 100644 --- a/packages/cypress/internal/install-cypress.js +++ b/packages/cypress/internal/install-cypress.js @@ -9,31 +9,42 @@ * repository rule while the file system remains read/write. */ -const {spawnSync, spawn} = require('child_process'); -const {readdirSync, statSync, writeFileSync, mkdirSync} = require('fs'); +const {spawnSync} = require('child_process'); +const {readdirSync, statSync, writeFileSync, mkdirSync, createWriteStream} = require('fs'); const { join, basename, relative, dirname, } = require('path'); + const nodePath = process.argv[1]; const cwd = process.cwd(); const cypressBin = process.argv[2]; -// Sandboxing doesn't work on windows, so we can just use the global cypress cache. -if (process.platform === 'win32') { - installGlobalCypressCache(); - process.exit(0); -} +const nodeModulesPath = join(cypressBin.split('node_modules')[0], 'node_modules'); +const tar = require(require.resolve('tar', { + paths: [ + join(nodeModulesPath, '@bazel', 'cypress', 'node_modules'), + nodeModulesPath, + ] +})); + +async function main() { + // Sandboxing doesn't work on windows, so we can just use the global cypress cache. + if (process.platform === 'win32') { + installGlobalCypressCache(); + process.exit(0); + } -// Attempt to install the cypress cache within the bazel sandbox and fallback to a global cypress -// cache as a last resort. -try { - installSandboxedCypressCache() -} catch (e) { - console.error('ERROR', e); - installGlobalCypressCache(); + // Attempt to install the cypress cache within the bazel sandbox and fallback to a global cypress + // cache as a last resort. + try { + await installSandboxedCypressCache() + } catch (e) { + console.error('ERROR', e); + installGlobalCypressCache(); + } } function installGlobalCypressCache() { @@ -70,11 +81,11 @@ cypress_web_test = _cypress_web_test`) } -function installSandboxedCypressCache() { - mkdirSync(join(cwd, 'cypress-cache')) +async function installSandboxedCypressCache() { + mkdirSync(join(cwd, 'cypress-install')) const env = { - CYPRESS_CACHE_FOLDER: join(cwd, 'cypress-cache'), + CYPRESS_CACHE_FOLDER: join(cwd, 'cypress-install'), PATH: `${dirname(nodePath)}:${process.env.PATH}`, DEBUG: 'cypress:*' } @@ -119,19 +130,24 @@ function installSandboxedCypressCache() { throw new Error(`cypress verify failed`); } + writeFileSync(join(env.CYPRESS_CACHE_FOLDER, 'bazel_cypress.json'), JSON.stringify({ + cypressExecutable: relative(cwd, CYPRESS_RUN_BINARY), + })); + const cacheFiles = []; walkDir(env.CYPRESS_CACHE_FOLDER, (filePath) => { - cacheFiles.push(filePath); + cacheFiles.push(relative(cwd, filePath)); }); + const archiveName = 'cypress.archive'; + await createCypressArchive(cacheFiles, join(cwd, archiveName)); + writeFileSync('index.bzl', `load( "//:packages/cypress/internal/cypress_web_test.bzl", _cypress_web_test = "cypress_web_test", ) cypress_web_test = _cypress_web_test`) - writeFileSync( - 'BUILD.bazel', - ` + writeFileSync('BUILD.bazel', ` package(default_visibility = ["//visibility:public"]) exports_files([ @@ -140,62 +156,32 @@ exports_files([ ]) filegroup( - name = "cypress_cache", - srcs = ${ - process.platform === 'darwin' ? - // On mac we are required to include cache files including spaces. These can only be - // included using a glob. - 'glob(["cypress-cache/**/*"]),' : - // On unix the only no files containing spaces are required to run cypress. - ` [ - ${ - cacheFiles.filter(f => !f.includes(' ')) - .map(f => `"${relative(cwd, f)}"`) - .join(',\n ')} - ]`} -) - -filegroup( - name = "cypress_executable", - srcs = ["${relative(cwd, CYPRESS_RUN_BINARY)}"] + name = "cypress_archive", + srcs = ["${relative(cwd, archiveName)}"], ) `.trim()) +} +function createCypressArchive(cypressFiles, archiveName) { + return new Promise((resolve, reject) => { + const writeStream = createWriteStream(archiveName); + + tar.create( + { + gzip: false, + portable: true, + noMtime: true, + }, + cypressFiles) + .pipe(writeStream) + .on('finish', (err) => { + if (err) { + return reject(err); + } + + return resolve(); + }) + }); +} - // On mac, the first run of cypress requires write access to the filesystem. - if (process.platform === 'darwin') { - const http = require('http'); - const server = http.createServer((_request, response) => { - response.writeHead(200, {'Content-Type': 'text/html'}); - response.write('hello-world\n'); - response.end(); - }) - .listen(0, '127.0.0.1'); - server.on('listening', () => { - const baseUrl = `http://127.0.0.1:${server.address().port}`; - writeFileSync( - 'cypress.json', JSON.stringify({baseUrl, 'integrationFolder': cwd, video: false})) - writeFileSync('spec.js', ` - describe('hello', () => { - it('should find hello', () => { - cy.visit('${baseUrl}'); - - cy.contains('hello'); - }); - }); - `) - - - spawn( - `${cypressBin}`, ['run', '--config-file=cypress.json', '--headless', '--spec=spec.js'], - spawnOptions) - .on('exit', (code) => { - server.close(); - - if (code !== 0) { - throw new Error('Failed to perform a dry-run of cypress') - } - }) - }) - } -} \ No newline at end of file +main() \ No newline at end of file diff --git a/packages/cypress/internal/run-cypress.js b/packages/cypress/internal/run-cypress.js index ebeeec6567..e6b562f0c8 100644 --- a/packages/cypress/internal/run-cypress.js +++ b/packages/cypress/internal/run-cypress.js @@ -1,21 +1,22 @@ const runfiles = require(process.env['BAZEL_NODE_RUNFILES_HELPER']); const init = require('cypress/lib/cli').init; const {join} = require('path'); +const {readFileSync} = require('fs'); -const [node, entry, configFilePath, pluginsFilePath, cypressExecutable, ...args] = process.argv; - -if (cypressExecutable) { - process.env.CYPRESS_RUN_BINARY = - join(process.cwd(), cypressExecutable.replace('external/', '../')); - process.env.CYPRESS_CACHE_FOLDER = - join(process.env.CYPRESS_RUN_BINARY.split('/cypress-cache/')[0], '/cypress-cache'); - process.env.HOME = process.env['TEST_TMPDIR']; -} +const [node, entry, configFilePath, pluginsFilePath, cypressTarPath, cypressBin, ...args] = + process.argv; const pluginsFile = runfiles.resolveWorkspaceRelative(pluginsFilePath).replace(process.cwd(), '.'); const configFile = runfiles.resolveWorkspaceRelative(configFilePath).replace(process.cwd(), '.'); -function invokeCypressWithCommand(command) { +async function invokeCypressWithCommand(command) { + process.env.HOME = process.env['TEST_TMPDIR']; + + if (cypressTarPath) { + const resolvedArchivePath = join(cypressTarPath.replace('external/', '../')); + await untarCypress(resolvedArchivePath, join(process.env['TEST_TMPDIR'])) + } + init([ node, entry, @@ -28,10 +29,61 @@ function invokeCypressWithCommand(command) { ]); } -// Detect that we are running as a test, by using well-known environment -// variables. See go/test-encyclopedia -if (!process.env.BUILD_WORKSPACE_DIRECTORY) { - invokeCypressWithCommand('run'); -} else { - invokeCypressWithCommand('open'); + + +function untarCypress(cypressTarPath, outputPath) { + return new Promise((resolve, reject) => { + const nodeModulesPath = join( + process.cwd(), cypressBin.replace('external/', '../').split('node_modules')[0], + 'node_modules'); + + const tar = require(require.resolve('tar', { + paths: [ + join(nodeModulesPath, '@bazel', 'cypress', 'node_modules'), + nodeModulesPath, + ] + })); + + + tar.x( + { + cwd: outputPath, + file: cypressTarPath, + noMtime: true, + }, + err => { + if (err) { + return reject(err); + } + + try { + const {cypressExecutable} = + JSON.parse(readFileSync(join(outputPath, 'cypress-install', 'bazel_cypress.json'))); + + process.env.CYPRESS_RUN_BINARY = join(outputPath, cypressExecutable); + process.env.CYPRESS_CACHE_FOLDER = outputPath; + } catch (err) { + return reject(err) + } + + return resolve(); + }) + }); +} + +async function main() { + try { + // Detect that we are running as a test, by using well-known environment + // variables. See go/test-encyclopedia + if (!process.env.BUILD_WORKSPACE_DIRECTORY) { + await invokeCypressWithCommand('run'); + } else { + await invokeCypressWithCommand('open'); + } + } catch (e) { + console.error(e); + process.exit(1) + } } + +main(); \ No newline at end of file diff --git a/packages/cypress/internal/template.cypress_web_test.bzl b/packages/cypress/internal/template.cypress_web_test.bzl index b5e984e774..77a371ab92 100644 --- a/packages/cypress/internal/template.cypress_web_test.bzl +++ b/packages/cypress/internal/template.cypress_web_test.bzl @@ -80,8 +80,10 @@ def cypress_web_test( plugins_file = Label("//plugins/base.js"), data = [], templated_args = [], - cypress_cache = Label("//:cypress_cache"), - cypress_executable = Label("//:cypress_executable"), + cypress = Label("TEMPLATED_node_modules_workspace_name//cypress"), + cypress_archive = Label("//:cypress_archive"), + cypress_bin = Label("TEMPLATED_node_modules_workspace_name//:node_modules/cypress/bin/cypress"), + tar = Label("TEMPLATED_node_modules_workspace_name//tar"), **kwargs): cypress_plugin = "{name}_cypress_plugin".format(name = name) tags = kwargs.pop("tags", []) + ["cypress"] @@ -101,8 +103,10 @@ def cypress_web_test( tags = tags, data = data + [ plugins_file, - cypress_cache, - cypress_executable, + cypress_archive, + cypress_bin, + cypress, + tar, "{cypress_plugin}".format(cypress_plugin = cypress_plugin), "{config_file}".format(config_file = config_file), ] + srcs, @@ -111,7 +115,8 @@ def cypress_web_test( "--nobazel_patch_module_resolver", "$(rootpath {config_file})".format(config_file = config_file), "$(rootpath {cypress_plugin})".format(cypress_plugin = cypress_plugin), - "$(rootpath {cypress_executable})".format(cypress_executable = cypress_executable), + "$(rootpath {cypress_archive})".format(cypress_archive = cypress_archive), + "$(rootpath {cypress_bin})".format(cypress_bin = cypress_bin), ] + templated_args, **kwargs ) diff --git a/packages/cypress/package.json b/packages/cypress/package.json index f502b0898e..e7465f6f4b 100644 --- a/packages/cypress/package.json +++ b/packages/cypress/package.json @@ -1,5 +1,8 @@ { "name": "@bazel/cypress", + "dependencies": { + "tar": "6.0.5" + }, "peerDependencies": { "cypress": ">=4.7.0" }, diff --git a/packages/cypress/test/package.json b/packages/cypress/test/package.json index d39ccd9cbb..ab90b3d4a8 100644 --- a/packages/cypress/test/package.json +++ b/packages/cypress/test/package.json @@ -6,6 +6,7 @@ "cypress": "^5.2.0", "@types/node": "14.0.14", "rxjs": "^6.5.2", - "typescript": "3.9.5" + "typescript": "3.9.5", + "tar": "6.0.5" } -} \ No newline at end of file +} diff --git a/packages/cypress/test/yarn.lock b/packages/cypress/test/yarn.lock index 1b703f0bc0..88cb73f8b6 100644 --- a/packages/cypress/test/yarn.lock +++ b/packages/cypress/test/yarn.lock @@ -287,6 +287,11 @@ check-more-types@^2.24.0: resolved "https://registry.yarnpkg.com/check-more-types/-/check-more-types-2.24.0.tgz#1420ffb10fd444dcfc79b43891bbfffd32a84600" integrity sha1-FCD/sQ/URNz8ebQ4kbv//TKoRgA= +chownr@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/chownr/-/chownr-2.0.0.tgz#15bfbe53d2eab4cf70f18a8cd68ebe5b3cb1dece" + integrity sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ== + ci-info@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-2.0.0.tgz#67a9e964be31a51e15e5010d58e6f12834002f46" @@ -740,6 +745,13 @@ fs-extra@^9.0.1: jsonfile "^6.0.1" universalify "^1.0.0" +fs-minipass@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/fs-minipass/-/fs-minipass-2.1.0.tgz#7f5036fdbf12c63c169190cbe4199c852271f9fb" + integrity sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg== + dependencies: + minipass "^3.0.0" + fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" @@ -1155,6 +1167,21 @@ minimist@^1.2.5: resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602" integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw== +minipass@^3.0.0: + version "3.1.3" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-3.1.3.tgz#7d42ff1f39635482e15f9cdb53184deebd5815fd" + integrity sha512-Mgd2GdMVzY+x3IJ+oHnVM+KG3lA5c8tnabyJKmHSaG2kAGpudxuOf8ToDkhumF7UzME7DecbQE9uOZhNm7PuJg== + dependencies: + yallist "^4.0.0" + +minizlib@^2.1.1: + version "2.1.2" + resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-2.1.2.tgz#e90d3466ba209b932451508a11ce3d3632145931" + integrity sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg== + dependencies: + minipass "^3.0.0" + yallist "^4.0.0" + mkdirp@^0.5.4: version "0.5.5" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def" @@ -1162,6 +1189,11 @@ mkdirp@^0.5.4: dependencies: minimist "^1.2.5" +mkdirp@^1.0.3: + version "1.0.4" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" + integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== + moment@^2.27.0: version "2.29.0" resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.0.tgz#fcbef955844d91deb55438613ddcec56e86a3425" @@ -1594,6 +1626,18 @@ symbol-observable@^1.1.0: resolved "https://registry.yarnpkg.com/symbol-observable/-/symbol-observable-1.2.0.tgz#c22688aed4eab3cdc2dfeacbb561660560a00804" integrity sha512-e900nM8RRtGhlV36KGEU9k65K3mPb1WV70OdjfxlG2EAuM1noi/E/BaW/uMhL7bPEssK8QV57vN3esixjUvcXQ== +tar@6.0.5: + version "6.0.5" + resolved "https://registry.yarnpkg.com/tar/-/tar-6.0.5.tgz#bde815086e10b39f1dcd298e89d596e1535e200f" + integrity sha512-0b4HOimQHj9nXNEAA7zWwMM91Zhhba3pspja6sQbgTpynOJf+bkjBnfybNYzbpLbnwXnbyB4LOREvlyXLkCHSg== + dependencies: + chownr "^2.0.0" + fs-minipass "^2.0.0" + minipass "^3.0.0" + minizlib "^2.1.1" + mkdirp "^1.0.3" + yallist "^4.0.0" + throttleit@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/throttleit/-/throttleit-1.0.0.tgz#9e785836daf46743145a5984b6268d828528ac6c" @@ -1733,6 +1777,11 @@ wrappy@1: resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8= +yallist@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" + integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== + yauzl@^2.10.0: version "2.10.0" resolved "https://registry.yarnpkg.com/yauzl/-/yauzl-2.10.0.tgz#c7eb17c93e112cb1086fa6d8e51fb0667b79a5f9"