feat: experimental single tab run mode for component testing (#23104)

* revive logic to run CT in a single tab * add feature flag: experimentalSingleTabRunMode * remove log * reset browser state between tests * document single tab run mode experiment; * add system test for experimental run mode * fix snapshots * use more simple project for testing * additional guard; * fix test * Apply suggestions from code review Co-authored-by: Emily Rohrbough <[email protected]> * destroy aut after each spec * update snapshot * fix types * add experiment flag error * add warning when using experimental flag with e2e * build binaries for experimentalSingleTabRunMode feature * build binaries take 2 * make error message more open ended * destroy AUT later in run mode lifecycle * add additional assertion around experimental flag * simplify error * remove test code from production code Co-authored-by: Emily Rohrbough <[email protected]>
cypress-io · Aug 16, 2022 · 91beb90 · 91beb90 · cypress-bot · Aug 16, 2022
1 parent f7dce39
commit 91beb90
Show file tree

Hide file tree

Showing 36 changed files with 551 additions and 46 deletions.
diff --git a/circle.yml b/circle.yml
@@ -27,8 +27,6 @@ mainBuildFilters: &mainBuildFilters
     branches:
       only:
         - develop
-        - reapply-state-refactor
-        - fix-or-skip-flaky-tests
 
 # usually we don't build Mac app - it takes a long time
 # but sometimes we want to really confirm we are doing the right thing
@@ -46,6 +44,7 @@ linuxArm64WorkflowFilters: &linux-arm64-workflow-filters
   when:
     or:
     - equal: [ develop, << pipeline.git.branch >> ]
+    - equal: [ "lmiller/experimental-single-tab-component-testing", << pipeline.git.branch >> ]
     - matches:
           pattern: "-release$"
           value: << pipeline.git.branch >>

diff --git a/cli/types/cypress.d.ts b/cli/types/cypress.d.ts
@@ -3062,6 +3062,11 @@ declare namespace Cypress {
   interface ComponentConfigOptions<ComponentDevServerOpts = any> extends Omit<CoreConfigOptions, 'baseUrl' | 'experimentalSessionAndOrigin'> {
     devServer: DevServerFn<ComponentDevServerOpts> | DevServerConfigOptions
     devServerConfig?: ComponentDevServerOpts
+    /**
+     * Runs all component specs in a single tab, trading spec isolation for faster run mode execution.
+     * @default false
+     */
+    experimentalSingleTabRunMode?: boolean
   }
 
   /**

diff --git a/packages/app/cypress/component/support/ctSupport.ts b/packages/app/cypress/component/support/ctSupport.ts
@@ -10,6 +10,17 @@ export const StubWebsocket = new Proxy<Socket>(Object.create(null), {
   },
 })
 
+beforeEach(() => {
+  if (!window.top?.getEventManager) {
+    throw Error('Could not find `window.top.getEventManager`. Expected `getEventManager` to be defined.')
+  }
+
+  // this is always undefined, since we only define it when
+  // running CT with a project that sets `experimentalSingleTabRunMode: true`
+  // @ts-ignore - dynamically defined during tests using
+  expect(window.top.getEventManager().autDestroyedCount).to.be.undefined
+})
+
 // Event manager with Cypress driver dependencies stubbed out
 // Useful for component testing
 export const createEventManager = () => {

diff --git a/packages/app/cypress/e2e/support/e2eSupport.ts b/packages/app/cypress/e2e/support/e2eSupport.ts
@@ -1,3 +1,10 @@
 import '@packages/frontend-shared/cypress/e2e/support/e2eSupport'
 import 'cypress-real-events/support'
 import './execute-spec'
+
+beforeEach(() => {
+  // this is always 0, since we only destroy the AUT when using
+  // `experimentalSingleTabRunMode, which is not a valid experiment for for e2e testing.
+  // @ts-ignore - dynamically defined during tests using
+  expect(window.top?.getEventManager().autDestroyedCount).to.be.undefined
+})
diff --git a/packages/app/src/runner/aut-iframe.ts b/packages/app/src/runner/aut-iframe.ts
@@ -35,6 +35,14 @@ export class AutIframe {
     return $iframe
   }
 
+  destroy () {
+    if (!this.$iframe) {
+      throw Error(`Cannot call #remove without first calling #create`)
+    }
+
+    this.$iframe.remove()
+  }
+
   showInitialBlankContents () {
     this._showContents(blankContents.initial())
   }

diff --git a/packages/app/src/runner/event-manager.ts b/packages/app/src/runner/event-manager.ts
@@ -11,6 +11,7 @@ import type { Socket } from '@packages/socket/lib/browser'
 import * as cors from '@packages/network/lib/cors'
 import { automation, useRunnerUiStore } from '../store'
 import { useScreenshotStore } from '../store/screenshot-store'
+import { getAutIframeModel } from '.'
 
 export type CypressInCypressMochaEvent = Array<Array<string | Record<string, any>>>
 
@@ -54,6 +55,8 @@ export class EventManager {
   studioRecorder: any
   selectorPlaygroundModel: any
   cypressInCypressMochaEvents: CypressInCypressMochaEvent[] = []
+  // Used for testing the experimentalSingleTabRunMode experiment. Ensures AUT is correctly destroyed between specs.
+  ws: Socket
 
   constructor (
     // import '@packages/driver'
@@ -64,10 +67,11 @@ export class EventManager {
     selectorPlaygroundModel: any,
     // StudioRecorder constructor
     StudioRecorderCtor: any,
-    private ws: Socket,
+    ws: Socket,
   ) {
     this.studioRecorder = new StudioRecorderCtor(this)
     this.selectorPlaygroundModel = selectorPlaygroundModel
+    this.ws = ws
   }
 
   getCypress () {
@@ -337,6 +341,13 @@ export class EventManager {
       rerun()
     })
 
+    this.ws.on('aut:destroy:init', () => {
+      const autIframe = getAutIframeModel()
+
+      autIframe.destroy()
+      this.ws.emit('aut:destroy:complete')
+    })
+
     // @ts-ignore
     const $window = this.$CypressDriver.$(window)
 

diff --git a/packages/config/__snapshots__/index.spec.ts.js b/packages/config/__snapshots__/index.spec.ts.js
@@ -39,6 +39,7 @@ exports['config/src/index .getDefaultValues returns list of public config keys 1
   "experimentalSessionAndOrigin": false,
   "experimentalModifyObstructiveThirdPartyCode": false,
   "experimentalSourceRewriting": false,
+  "experimentalSingleTabRunMode": false,
   "fileServerFolder": "",
   "fixturesFolder": "cypress/fixtures",
   "excludeSpecPattern": "*.hot-update.js",
@@ -120,6 +121,7 @@ exports['config/src/index .getDefaultValues returns list of public config keys f
   "experimentalSessionAndOrigin": false,
   "experimentalModifyObstructiveThirdPartyCode": false,
   "experimentalSourceRewriting": false,
+  "experimentalSingleTabRunMode": false,
   "fileServerFolder": "",
   "fixturesFolder": "cypress/fixtures",
   "excludeSpecPattern": "*.hot-update.js",
@@ -197,6 +199,7 @@ exports['config/src/index .getPublicConfigKeys returns list of public config key
   "experimentalSessionAndOrigin",
   "experimentalModifyObstructiveThirdPartyCode",
   "experimentalSourceRewriting",
+  "experimentalSingleTabRunMode",
   "fileServerFolder",
   "fixturesFolder",
   "excludeSpecPattern",

diff --git a/packages/config/src/browser.ts b/packages/config/src/browser.ts
@@ -75,7 +75,7 @@ export function resetIssuedWarnings () {
   issuedWarnings.clear()
 }
 
-const validateNoBreakingOptions = (breakingCfgOptions: BreakingOption[], cfg: any, onWarning: ErrorHandler, onErr: ErrorHandler, testingType?: TestingType) => {
+const validateNoBreakingOptions = (breakingCfgOptions: Readonly<BreakingOption[]>, cfg: any, onWarning: ErrorHandler, onErr: ErrorHandler, testingType?: TestingType) => {
   breakingCfgOptions.forEach(({ name, errorKey, newName, isWarning, value }) => {
     if (_.has(cfg, name)) {
       if (value && cfg[name] !== value) {

diff --git a/packages/config/src/options.ts b/packages/config/src/options.ts
@@ -3,31 +3,37 @@ import path from 'path'
 
 // @ts-ignore
 import pkg from '@packages/root'
+import type { AllCypressErrorNames } from '@packages/errors'
 import type { TestingType } from '@packages/types'
 
 import * as validate from './validation'
 
-export type BreakingOptionErrorKey =
-  | 'COMPONENT_FOLDER_REMOVED'
-  | 'INTEGRATION_FOLDER_REMOVED'
-  | 'CONFIG_FILE_INVALID_ROOT_CONFIG'
-  | 'CONFIG_FILE_INVALID_ROOT_CONFIG_E2E'
-  | 'CONFIG_FILE_INVALID_ROOT_CONFIG_COMPONENT'
-  | 'CONFIG_FILE_INVALID_TESTING_TYPE_CONFIG_COMPONENT'
-  | 'CONFIG_FILE_INVALID_TESTING_TYPE_CONFIG_E2E'
-  | 'EXPERIMENTAL_COMPONENT_TESTING_REMOVED'
-  | 'EXPERIMENTAL_SAMESITE_REMOVED'
-  | 'EXPERIMENTAL_NETWORK_STUBBING_REMOVED'
-  | 'EXPERIMENTAL_RUN_EVENTS_REMOVED'
-  | 'EXPERIMENTAL_SESSION_SUPPORT_REMOVED'
-  | 'EXPERIMENTAL_SHADOW_DOM_REMOVED'
-  | 'EXPERIMENTAL_STUDIO_REMOVED'
-  | 'FIREFOX_GC_INTERVAL_REMOVED'
-  | 'NODE_VERSION_DEPRECATION_SYSTEM'
-  | 'NODE_VERSION_DEPRECATION_BUNDLED'
-  | 'PLUGINS_FILE_CONFIG_OPTION_REMOVED'
-  | 'RENAMED_CONFIG_OPTION'
-  | 'TEST_FILES_RENAMED'
+const BREAKING_OPTION_ERROR_KEY: Readonly<AllCypressErrorNames[]> = [
+  'COMPONENT_FOLDER_REMOVED',
+  'INTEGRATION_FOLDER_REMOVED',
+  'CONFIG_FILE_INVALID_ROOT_CONFIG',
+  'CONFIG_FILE_INVALID_ROOT_CONFIG_E2E',
+  'CONFIG_FILE_INVALID_ROOT_CONFIG_COMPONENT',
+  'CONFIG_FILE_INVALID_TESTING_TYPE_CONFIG_COMPONENT',
+  'CONFIG_FILE_INVALID_TESTING_TYPE_CONFIG_E2E',
+  'EXPERIMENTAL_COMPONENT_TESTING_REMOVED',
+  'EXPERIMENTAL_SAMESITE_REMOVED',
+  'EXPERIMENTAL_NETWORK_STUBBING_REMOVED',
+  'EXPERIMENTAL_RUN_EVENTS_REMOVED',
+  'EXPERIMENTAL_SESSION_SUPPORT_REMOVED',
+  'EXPERIMENTAL_SINGLE_TAB_RUN_MODE',
+  'EXPERIMENTAL_SHADOW_DOM_REMOVED',
+  'EXPERIMENTAL_STUDIO_REMOVED',
+  'EXPERIMENTAL_STUDIO_REMOVED',
+  'FIREFOX_GC_INTERVAL_REMOVED',
+  'NODE_VERSION_DEPRECATION_SYSTEM',
+  'NODE_VERSION_DEPRECATION_BUNDLED',
+  'PLUGINS_FILE_CONFIG_OPTION_REMOVED',
+  'RENAMED_CONFIG_OPTION',
+  'TEST_FILES_RENAMED',
+] as const
+
+export type BreakingOptionErrorKey = typeof BREAKING_OPTION_ERROR_KEY[number]
 
 export type OverrideLevel = 'any' | 'suite' | 'never'
 
@@ -211,6 +217,12 @@ const driverConfigOptions: Array<DriverConfigOption> = [
     validation: validate.isBoolean,
     isExperimental: true,
     requireRestartOnChange: 'server',
+  }, {
+    name: 'experimentalSingleTabRunMode',
+    defaultValue: false,
+    validation: validate.isBoolean,
+    isExperimental: true,
+    requireRestartOnChange: 'server',
   }, {
     name: 'fileServerFolder',
     defaultValue: '',
@@ -520,7 +532,7 @@ export const options: Array<DriverConfigOption | RuntimeConfigOption> = [
 /**
  * Values not allowed in 10.X+ in the root, e2e and component config
  */
-export const breakingOptions: Array<BreakingOption> = [
+export const breakingOptions: Readonly<BreakingOption[]> = [
   {
     name: 'blacklistHosts',
     errorKey: 'RENAMED_CONFIG_OPTION',
@@ -592,7 +604,7 @@ export const breakingOptions: Array<BreakingOption> = [
     newName: 'specPattern',
     isWarning: false,
   },
-]
+] as const
 
 export const breakingRootOptions: Array<BreakingOption> = [
   {
@@ -645,6 +657,12 @@ export const breakingRootOptions: Array<BreakingOption> = [
 
 export const testingTypeBreakingOptions: { e2e: Array<BreakingOption>, component: Array<BreakingOption> } = {
   e2e: [
+    {
+      name: 'experimentalSingleTabRunMode',
+      errorKey: 'EXPERIMENTAL_SINGLE_TAB_RUN_MODE',
+      isWarning: false,
+      testingTypes: ['e2e'],
+    },
     {
       name: 'indexHtmlFile',
       errorKey: 'CONFIG_FILE_INVALID_TESTING_TYPE_CONFIG_E2E',

diff --git a/packages/errors/__snapshot-html__/COMPONENT_TESTING_MISMATCHED_DEPENDENCIES.html b/packages/errors/__snapshot-html__/COMPONENT_TESTING_MISMATCHED_DEPENDENCIES.html
diff --git a/packages/errors/__snapshot-html__/EXPERIMENTAL_SINGLE_TAB_RUN_MODE.html b/packages/errors/__snapshot-html__/EXPERIMENTAL_SINGLE_TAB_RUN_MODE.html
diff --git a/packages/errors/src/errors.ts b/packages/errors/src/errors.ts
@@ -1085,6 +1085,12 @@ export const AllCypressErrors = {
 
         You can safely remove the ${fmt.highlight(`experimentalStudio`)} configuration option from your config.`
   },
+  EXPERIMENTAL_SINGLE_TAB_RUN_MODE: () => {
+    return errTemplate`\
+        The ${fmt.highlight(`experimentalSingleTabRunMode`)} experiment is currently only supported for Component Testing.
+
+        If you have feedback about the experiment, please join the discussion here: http://on.cypress.io/single-tab-run-mode`
+  },
   FIREFOX_GC_INTERVAL_REMOVED: () => {
     return errTemplate`\
         The ${fmt.highlight(`firefoxGcInterval`)} configuration option was removed in ${fmt.cypressVersion(`8.0.0`)}. It was introduced to work around a bug in Firefox 79 and below.

diff --git a/packages/errors/test/unit/visualSnapshotErrors_spec.ts b/packages/errors/test/unit/visualSnapshotErrors_spec.ts
@@ -1184,7 +1184,7 @@ describe('visual error templates', () => {
                 package: 'vite',
                 installer: 'vite',
                 description: 'Vite is dev server that serves your source files over native ES modules',
-                minVersion: '>=2.0.0',
+                minVersion: '^=2.0.0 || ^=3.0.0',
               },
               satisfied: false,
               detectedVersion: '1.0.0',
@@ -1194,5 +1194,11 @@ describe('visual error templates', () => {
         ],
       }
     },
+
+    EXPERIMENTAL_SINGLE_TAB_RUN_MODE: () => {
+      return {
+        default: [],
+      }
+    },
   })
 })
diff --git a/packages/frontend-shared/src/gql-components/error/BaseError.cy.tsx b/packages/frontend-shared/src/gql-components/error/BaseError.cy.tsx
@@ -4,7 +4,7 @@ import { BaseErrorFragmentDoc } from '../../../../launchpad/src/generated/graphq
 import dedent from 'dedent'
 
 // Selectors
-const headerSelector = 'h1[data-testid=error-header]'
+const headerSelector = 'h1[data-cy=error-header]'
 const messageSelector = '[data-testid=error-message]'
 const retryButtonSelector = 'button[data-testid=error-retry-button]'
 const docsButtonSelector = 'a[data-testid=error-docs-button]'

diff --git a/packages/frontend-shared/src/gql-components/error/BaseError.vue b/packages/frontend-shared/src/gql-components/error/BaseError.vue
@@ -8,7 +8,7 @@
         <h1
           v-if="baseError.title"
           class="font-medium leading-snug text-32px text-gray-900"
-          data-testid="error-header"
+          data-cy="error-header"
         >
           <slot name="header">
             {{ baseError.title }}

diff --git a/packages/launchpad/cypress.config.ts b/packages/launchpad/cypress.config.ts
@@ -20,6 +20,7 @@ export default defineConfig({
     videoCompression: false, // turn off video compression for CI
   },
   component: {
+    experimentalSingleTabRunMode: true,
     supportFile: 'cypress/component/support/index.ts',
     devServer: {
       bundler: 'vite',

diff --git a/packages/launchpad/cypress/e2e/config-warning.cy.ts b/packages/launchpad/cypress/e2e/config-warning.cy.ts
@@ -67,6 +67,26 @@ describe('baseUrl', () => {
   })
 })
 
+describe('experimentalSingleTabRunMode', () => {
+  it('is a valid config for component testing', () => {
+    cy.scaffoldProject('experimentalSingleTabRunMode')
+    cy.openProject('experimentalSingleTabRunMode')
+    cy.visitLaunchpad()
+    cy.get('[data-cy-testingtype="component"]').click()
+    cy.get('h1').contains('Initializing Config').should('not.exist')
+    cy.get('h1').contains('Choose a Browser')
+  })
+
+  it('is not a valid config for e2e testing', () => {
+    cy.scaffoldProject('experimentalSingleTabRunMode')
+    cy.openProject('experimentalSingleTabRunMode')
+    cy.visitLaunchpad()
+    cy.get('[data-cy-testingtype="e2e"]').click()
+    cy.findByTestId('error-header').contains('Cypress configuration error')
+    cy.findByTestId('alert-body').contains('The experimentalSingleTabRunMode experiment is currently only supported for Component Testing.')
+  })
+})
+
 describe('experimentalStudio', () => {
   it('should show experimentalStudio warning if Cypress detects experimentalStudio config has been set', () => {
     cy.scaffoldProject('experimental-studio')