iterate through specs in parallel (#2154)

- fixes: #2153
- fixes: #1566
- fixes: #1690
- fixes: #2275
- fixes: #2276

circle.yml

@@ -171,7 +171,9 @@ jobs:
           paths:
             - packages/ts/node_modules
 
-      - run: npm run stop-only
+        ## TODO: this needs to be reenabled when
+        ## we update stop-only
+    #   - run: npm run stop-only
       ## now go build all of subpackages
       - run: npm run build
 

cli/__snapshots__/cli_spec.js

@@ -67,6 +67,9 @@ exports['shows help for run --foo 1'] = `
     -c, --config <config>                      sets configuration values. separate multiple values with a comma. overrides any value in cypress.json.
     -b, --browser <browser-name>               runs Cypress in the browser with the given name. note: using an external browser will not record a video.
     -P, --project <project-path>               path to the project
+    --parallel                                 enables concurrent runs and automatic load balancing of specs across multiple machines or processes
+    --group <name>                             a named group for recorded runs in the Cypress dashboard
+    --ci-build-id <id>                         the unique identifier for a run on your CI provider. typically a "BUILD_ID" env var. this value is automatically detected for most CI providers
     --no-exit                                  keep the browser open after tests finish
     --dev                                      runs cypress in development and bypasses binary check
     -h, --help                                 output usage information
@@ -94,14 +97,14 @@ exports['cli help command shows help 1'] = `
 
   Options:
 
-    -v, --version  Prints Cypress version
+    -v, --version  prints Cypress version
     -h, --help     output usage information
 
 
   Commands:
 
     help                Shows CLI help and exits
-    version             Prints Cypress version
+    version             prints Cypress version
     run [options]       Runs Cypress tests from the CLI without the GUI
     open [options]      Opens Cypress in the interactive GUI.
     install [options]   Installs the Cypress executable matching this package's version
@@ -131,14 +134,14 @@ exports['cli help command shows help for -h 1'] = `
 
   Options:
 
-    -v, --version  Prints Cypress version
+    -v, --version  prints Cypress version
     -h, --help     output usage information
 
 
   Commands:
 
     help                Shows CLI help and exits
-    version             Prints Cypress version
+    version             prints Cypress version
     run [options]       Runs Cypress tests from the CLI without the GUI
     open [options]      Opens Cypress in the interactive GUI.
     install [options]   Installs the Cypress executable matching this package's version
@@ -168,14 +171,14 @@ exports['cli help command shows help for --help 1'] = `
 
   Options:
 
-    -v, --version  Prints Cypress version
+    -v, --version  prints Cypress version
     -h, --help     output usage information
 
 
   Commands:
 
     help                Shows CLI help and exits
-    version             Prints Cypress version
+    version             prints Cypress version
     run [options]       Runs Cypress tests from the CLI without the GUI
     open [options]      Opens Cypress in the interactive GUI.
     install [options]   Installs the Cypress executable matching this package's version
@@ -207,14 +210,14 @@ exports['cli unknown command shows usage and exits 1'] = `
 
     Options:
 
-      -v, --version  Prints Cypress version
+      -v, --version  prints Cypress version
       -h, --help     output usage information
 
 
     Commands:
 
       help                Shows CLI help and exits
-      version             Prints Cypress version
+      version             prints Cypress version
       run [options]       Runs Cypress tests from the CLI without the GUI
       open [options]      Opens Cypress in the interactive GUI.
       install [options]   Installs the Cypress executable matching this package's version

cli/lib/cli.js

@@ -28,7 +28,7 @@ const parseOpts = (opts) => {
     'project', 'spec', 'reporter', 'reporterOptions', 'path', 'destination',
     'port', 'env', 'cypressVersion', 'config', 'record', 'key',
     'browser', 'detached', 'headed', 'global', 'dev', 'force', 'exit',
-    'cachePath', 'cacheList', 'cacheClear'
+    'cachePath', 'cacheList', 'cacheClear', 'parallel', 'group', 'ciBuildId'
   )
 
   if (opts.exit) {
@@ -56,14 +56,17 @@ const descriptions = {
   detached: 'runs Cypress application in detached mode',
   project: 'path to the project',
   global: 'force Cypress into global mode as if its globally installed',
-  version: 'Prints Cypress version',
+  version: 'prints Cypress version',
   headed: 'displays the Electron browser instead of running headlessly',
   dev: 'runs cypress in development and bypasses binary check',
   forceInstall: 'force install the Cypress binary',
   exit: 'keep the browser open after tests finish',
   cachePath: 'print the cypress binary cache path',
   cacheList: 'list the currently cached versions',
   cacheClear: 'delete the Cypress binary cache',
+  group: 'a named group for recorded runs in the Cypress dashboard',
+  parallel: 'enables concurrent runs and automatic load balancing of specs across multiple machines or processes',
+  ciBuildId: 'the unique identifier for a run on your CI provider. typically a "BUILD_ID" env var. this value is automatically detected for most CI providers',
 }
 
 const knownCommands = ['version', 'run', 'open', 'install', 'verify', '-v', '--version', 'help', '-h', '--help', 'cache']
@@ -134,6 +137,9 @@ module.exports = {
     .option('-c, --config <config>', text('config'))
     .option('-b, --browser <browser-name>', text('browser'))
     .option('-P, --project <project-path>', text('project'))
+    .option('--parallel', text('parallel'))
+    .option('--group <name>', text('group'))
+    .option('--ci-build-id <id>', text('ciBuildId'))
     .option('--no-exit', text('exit'))
     .option('--dev', text('dev'), coerceFalse)
     .action((opts) => {

cli/lib/exec/run.js

@@ -64,6 +64,18 @@ const processRunOptions = (options = {}) => {
     args.push('--record', options.record)
   }
 
+  if (options.parallel) {
+    args.push('--parallel')
+  }
+
+  if (options.group) {
+    args.push('--group', options.group)
+  }
+
+  if (options.ciBuildId) {
+    args.push('--ci-build-id', options.ciBuildId)
+  }
+
   if (options.outputPath) {
     args.push('--output-path', options.outputPath)
   }

cli/test/lib/cli_spec.js

@@ -205,6 +205,20 @@ describe('cli', function () {
       expect(run.start).to.be.calledWith({ exit: false })
     })
 
+    it('calls run with --parallel', function () {
+      this.exec('run --parallel')
+      expect(run.start).to.be.calledWith({ parallel: true })
+    })
+
+    it('calls runs with --ci-build-id', function () {
+      this.exec('run --ci-build-id 123')
+      expect(run.start).to.be.calledWith({ ciBuildId: '123' })
+    })
+
+    it('calls runs with --group', function () {
+      this.exec('run --group staging')
+      expect(run.start).to.be.calledWith({ group: 'staging' })
+    })
   })
 
   context('cypress open', function () {

package.json

@@ -26,11 +26,10 @@
     "lint-coffee": "coffeelint scripts/**/*.coffee",
     "lint": "npm run lint-js && npm run lint-coffee",
     "pretest": "npm run lint && npm run all lint && npm run test-scripts",
-    "precommit": "lint-staged; npm run warn-only",
+    "precommit": "lint-staged",
     "precommit-lint": "eslint --fix",
-    "prepush": "npm run stop-only",
-    "stop-only": "stop-only packages --exclude-dir .cy --exclude-dir .projects --exclude-dir node_modules --exclude-dir dist --exclude-dir dist-test --exclude-dir fixtures --exclude-dir lib --exclude-dir bower_components",
-    "warn-only": "stop-only --warn packages --exclude-dir .cy --exclude-dir .projects --exclude-dir node_modules --exclude-dir dist --exclude-dir dist-test --exclude-dir fixtures --exclude-dir lib --exclude-dir bower_components",
+    "stop-only": "stop-only --folder packages --skip .cy,.publish,.projects,node_modules,dist,dist-test,fixtures,lib,bower_components,spec_helper.coffee",
+    "warn-only": "stop-only --warn packages --skip .cy,.publish,.projects,node_modules,dist,dist-test,fixtures,lib,bower_components,spec_helper.coffee",
     "bump": "node ./scripts/binary.js bump",
     "set-next-ci-version": "node ./scripts/binary.js setNextVersion",
     "binary-build": "node ./scripts/binary.js build",
@@ -80,7 +79,7 @@
     "execa": "^0.8.0",
     "execa-wrap": "^1.1.0",
     "filesize": "^3.5.10",
-    "fs-extra": "^2.1.2",
+    "fs-extra": "^7.0.0",
     "gift": "^0.10.0",
     "gulp": "^3.9.1",
     "gulp-awspublish": "^3.3.0",
@@ -108,8 +107,8 @@
     "print-arch": "^1.0.0",
     "ramda": "^0.24.1",
     "shelljs": "^0.7.8",
-    "snap-shot-it": "^4.0.1",
-    "stop-only": "1.2.1",
+    "snap-shot-it": "^5.0.1",
+    "stop-only": "2.1.0",
     "terminal-banner": "^1.0.0",
     "typescript": "^2.3.4",
     "vagrant": "0.0.1",

packages/server/__snapshots__/async_timeouts_spec.coffee → packages/server/__snapshots__/async_timeouts_spec.coffee.js

@@ -1,4 +1,5 @@
 exports['e2e async timeouts failing1 1'] = `
+
 ====================================================================================================
 
   (Run Starting)
@@ -67,5 +68,5 @@ exports['e2e async timeouts failing1 1'] = `
   └────────────────────────────────────────────────────────────────────────────────────────────────┘
     1 of 1 failed (100%)                        XX:XX        1        -        1        -        -  
 
-`
 
+`

packages/server/__snapshots__/base_url_spec.coffee → packages/server/__snapshots__/base_url_spec.coffee.js

@@ -1,4 +1,5 @@
 exports['e2e baseUrl https passes 1'] = `
+
 ====================================================================================================
 
   (Run Starting)
@@ -55,9 +56,11 @@ exports['e2e baseUrl https passes 1'] = `
   └────────────────────────────────────────────────────────────────────────────────────────────────┘
     All specs passed!                           XX:XX        1        1        -        -        -  
 
+
 `
 
 exports['e2e baseUrl http passes 1'] = `
+
 ====================================================================================================
 
   (Run Starting)
@@ -114,5 +117,5 @@ exports['e2e baseUrl http passes 1'] = `
   └────────────────────────────────────────────────────────────────────────────────────────────────┘
     All specs passed!                           XX:XX        1        1        -        -        -  
 
-`
 
+`

packages/server/__snapshots__/blacklist_hosts_spec.coffee → packages/server/__snapshots__/blacklist_hosts_spec.coffee.js

@@ -1,4 +1,5 @@
 exports['e2e blacklist passes 1'] = `
+
 ====================================================================================================
 
   (Run Starting)
@@ -55,5 +56,5 @@ exports['e2e blacklist passes 1'] = `
   └────────────────────────────────────────────────────────────────────────────────────────────────┘
     All specs passed!                           XX:XX        1        1        -        -        -  
 
-`
 
+`

packages/server/__snapshots__/browserify_babel_es2015_spec.coffee → packages/server/__snapshots__/browserify_babel_es2015_spec.coffee.js

@@ -1,4 +1,5 @@
 exports['e2e browserify, babel, es2015 passes 1'] = `
+
 ====================================================================================================
 
   (Run Starting)
@@ -57,9 +58,11 @@ exports['e2e browserify, babel, es2015 passes 1'] = `
   └────────────────────────────────────────────────────────────────────────────────────────────────┘
     All specs passed!                           XX:XX        3        3        -        -        -  
 
+
 `
 
 exports['e2e browserify, babel, es2015 fails 1'] = `
+
 ====================================================================================================
 
   (Run Starting)
@@ -88,7 +91,6 @@ SyntaxError: /foo/bar/.projects/e2e/lib/fail.js: Unexpected token (2:0)
 > 2 | 
     | ^ while parsing file: /foo/bar/.projects/e2e/lib/fail.js
 
-
 This occurred while Cypress was compiling and bundling your test code. This is usually caused by:
 
 - A missing file or dependency
@@ -128,5 +130,5 @@ Fix the error in your code and re-run your tests.
   └────────────────────────────────────────────────────────────────────────────────────────────────┘
     1 of 1 failed (100%)                        XX:XX        -        -        1        -        -  
 
-`
 
+`

packages/server/__snapshots__/busted_support_file_spec.coffee → packages/server/__snapshots__/busted_support_file_spec.coffee.js

@@ -1,4 +1,5 @@
 exports['e2e busted support file passes 1'] = `
+
 ====================================================================================================
 
   (Run Starting)
@@ -22,7 +23,6 @@ The error was:
 
 Error: Cannot find module './does/not/exist' from '/foo/bar/.projects/busted-support-file/cypress/support'
 
-
 This occurred while Cypress was compiling and bundling your test code. This is usually caused by:
 
 - A missing file or dependency
@@ -62,5 +62,5 @@ Fix the error in your code and re-run your tests.
   └────────────────────────────────────────────────────────────────────────────────────────────────┘
     1 of 1 failed (100%)                        XX:XX        -        -        1        -        -  
 
-`
 
+`

packages/server/__snapshots__/cache_spec.coffee → packages/server/__snapshots__/cache_spec.coffee.js

@@ -1,4 +1,5 @@
 exports['e2e cache passes 1'] = `
+
 ====================================================================================================
 
   (Run Starting)
@@ -58,5 +59,5 @@ exports['e2e cache passes 1'] = `
   └────────────────────────────────────────────────────────────────────────────────────────────────┘
     All specs passed!                           XX:XX        4        4        -        -        -  
 
-`
 
+`

packages/server/__snapshots__/caught_uncaught_hook_errors_spec.coffee → packages/server/__snapshots__/caught_uncaught_hook_errors_spec.coffee.js

@@ -1,4 +1,5 @@
 exports['e2e caught and uncaught hooks errors failing1 1'] = `
+
 ====================================================================================================
 
   (Run Starting)
@@ -111,9 +112,11 @@ Because this error occurred during a 'before all' hook we are skipping the remai
   └────────────────────────────────────────────────────────────────────────────────────────────────┘
     1 of 1 failed (100%)                        XX:XX       11        5        3        -        3  
 
+
 `
 
 exports['e2e caught and uncaught hooks errors failing2 1'] = `
+
 ====================================================================================================
 
   (Run Starting)
@@ -198,9 +201,11 @@ Because this error occurred during a 'before each' hook we are skipping the rema
   └────────────────────────────────────────────────────────────────────────────────────────────────┘
     1 of 1 failed (100%)                        XX:XX        7        4        1        -        2  
 
+
 `
 
 exports['e2e caught and uncaught hooks errors failing3 1'] = `
+
 ====================================================================================================
 
   (Run Starting)
@@ -277,9 +282,11 @@ Because this error occurred during a 'before each' hook we are skipping all of t
   └────────────────────────────────────────────────────────────────────────────────────────────────┘
     1 of 1 failed (100%)                        XX:XX        4        -        1        -        3  
 
+
 `
 
 exports['e2e caught and uncaught hooks errors failing4 1'] = `
+
 ====================================================================================================
 
   (Run Starting)
@@ -362,5 +369,5 @@ Because this error occurred during a 'before each' hook we are skipping the rema
   └────────────────────────────────────────────────────────────────────────────────────────────────┘
     1 of 1 failed (100%)                        XX:XX        3        2        1        -        -  
 
-`
 
+`

packages/server/__snapshots__/commands_outside_of_test_spec.coffee → packages/server/__snapshots__/commands_outside_of_test_spec.coffee.js

@@ -1,4 +1,5 @@
 exports['e2e commands outside of test fails 1'] = `
+
 ====================================================================================================
 
   (Run Starting)
@@ -87,5 +88,5 @@ We dynamically generated a new test to display this failure.
   └────────────────────────────────────────────────────────────────────────────────────────────────┘
     1 of 1 failed (100%)                        XX:XX        1        -        1        -        -  
 
-`
 
+`