From 531fa0a3342311a34a245d909cfb86ee37baede2 Mon Sep 17 00:00:00 2001 From: Dan Bjorge Date: Thu, 23 May 2019 15:43:02 -0700 Subject: [PATCH] use jest-circus runner to improve beforeAll/beforeEach failure behavior (#12) Updates jest.config.js to recommend use of the jest-circus runner to enable tests to early-exit when beforeAll/beforeEach methods fail, per facebook/jest#2713 --- typescript-selenium-webdriver/jest.config.js | 10 ++ typescript-selenium-webdriver/package.json | 1 + .../tests/index.test.ts | 152 +++++++++--------- typescript-selenium-webdriver/yarn.lock | 22 +++ 4 files changed, 109 insertions(+), 76 deletions(-) diff --git a/typescript-selenium-webdriver/jest.config.js b/typescript-selenium-webdriver/jest.config.js index 00fb1283..0822de2c 100644 --- a/typescript-selenium-webdriver/jest.config.js +++ b/typescript-selenium-webdriver/jest.config.js @@ -1,8 +1,18 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. module.exports = { + // These are necessary to use typescript with Jest tests preset:'ts-jest', testEnvironment: 'node', + + // Using the jest-circus testRunner ensures that tests will not run if beforeAll or beforeEach fails. + // + // Since browser automation tests commonly use beforeAll/beforeEach to perform browser/page setup, + // it's usually misleading to allow tests to continue trying to execute when they have failed. + // + // See https://github.com/facebook/jest/issues/2713 for details. + testRunner: 'jest-circus/runner', + reporters: [ // This enables console output for local development and build log output in Pipelines. 'default', diff --git a/typescript-selenium-webdriver/package.json b/typescript-selenium-webdriver/package.json index ebd4bcdc..fdc7c8d8 100644 --- a/typescript-selenium-webdriver/package.json +++ b/typescript-selenium-webdriver/package.json @@ -12,6 +12,7 @@ "axe-webdriverjs": "^2.2.0", "http-server": "^0.11.1", "jest": "^24.8.0", + "jest-circus": "^24.8.0", "jest-junit": "^6.4.0", "selenium-webdriver": "^4.0.0-alpha.1", "ts-jest": "^24.0.2", diff --git a/typescript-selenium-webdriver/tests/index.test.ts b/typescript-selenium-webdriver/tests/index.test.ts index 33617fea..51232a8a 100644 --- a/typescript-selenium-webdriver/tests/index.test.ts +++ b/typescript-selenium-webdriver/tests/index.test.ts @@ -1,76 +1,76 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. -import { convertAxeToSarif } from 'axe-sarif-converter'; -import * as AxeBuilder from 'axe-webdriverjs'; -import * as fs from 'fs'; -import * as path from 'path'; -import { Builder, By, ThenableWebDriver, until } from 'selenium-webdriver'; -import * as chrome from 'selenium-webdriver/chrome'; -import { promisify } from 'util'; - -describe('index.html', () => { - let driver: ThenableWebDriver; - - // Starting a browser instance is time-consuming, so we share one browser instance between - // all tests in the file (by initializing it in beforeAll rather than beforeEach) - beforeAll(async () => { - // The default timeout (5 seconds) is not always enough to start/quit a browser instance - jest.setTimeout(30000); - - // This is for the benefit of the Azure Pipelines Hosted Windows agents, which come with - // webdrivers preinstalled but not on the PATH where Selenium looks for them by default. - // See https://docs.microsoft.com/en-us/azure/devops/pipelines/test/continuous-test-selenium#decide-how-you-will-deploy-and-test-your-app - if (process.env.ChromeWebDriver) { - const hostedAgentChromedriverPath = path.join(process.env.ChromeWebDriver, 'chromedriver.exe'); - const chromeService = new chrome.ServiceBuilder(hostedAgentChromedriverPath).build(); - chrome.setDefaultService(chromeService); - } - - // Selenium supports many browsers, not just Chrome. - // See https://www.npmjs.com/package/selenium-webdriver for examples. - driver = new Builder() - .forBrowser('chrome') - .setChromeOptions(new chrome.Options().headless()) - .build(); - }); - - afterAll(async () => { - await driver.quit(); - }); - - beforeEach(async () => { - // For simplicity, we're pointing our test browser directly to a static html file on disk. - // - // In a real project, you would probably use a localhost http server (Express.js, for example) - // and point selenium-webdriver to a http://localhost link. - // - // See https://jestjs.io/docs/en/testing-frameworks for examples. - const pageUnderTest = 'file://' + path.join(__dirname, '..', 'src', 'index.html'); - await driver.get(pageUnderTest); - - // Checking for a known element on the page in beforeEach serves two purposes: - // * It acts as a sanity check that our browser automation setup basically works - // * It ensures that the page is loaded before we run our accessibility scans - await driver.wait(until.elementLocated(By.css('h1'))); - }); - - it('only contains known accessibility violations', async () => { - // Run an accessibility scan using axe-webdriverjs - const axeResults = await AxeBuilder(driver).analyze(); - - // Write a test expectation that accounts for "known" issues we want to baseline - expect(axeResults.violations.length).toBe(3); - - // Write the axe results to a .sarif file, so we can use the SARIF Multitool to - // apply a baseline file and show the results in the Scans tab in Azure Pipelines - const sarifResults = convertAxeToSarif(axeResults); - const testResultsDirectory = path.join(__dirname, '..', 'test-results'); - await promisify(fs.mkdir)(testResultsDirectory, { recursive: true }); - await promisify(fs.writeFile)( - path.join(testResultsDirectory, 'index.html.axe-core.sarif'), - // We'll be checking in the resulting .sarif file for baselining purposes, so - // it's a good idea to use a spacing argument (here, "2") to pretty-print the - // JSON. This makes it much more pleasant to diff when it changes. - JSON.stringify(sarifResults, null, 2)); - }); -}); +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +import { convertAxeToSarif } from 'axe-sarif-converter'; +import * as AxeBuilder from 'axe-webdriverjs'; +import * as fs from 'fs'; +import * as path from 'path'; +import { Builder, By, ThenableWebDriver, until } from 'selenium-webdriver'; +import * as chrome from 'selenium-webdriver/chrome'; +import { promisify } from 'util'; + +// The default timeout for tests/fixtures (5 seconds) is not always enough to start/quit/navigate a browser instance. +const TEST_TIMEOUT_MS = 30000; + +describe('index.html', () => { + let driver: ThenableWebDriver; + + // Starting a browser instance is time-consuming, so we share one browser instance between + // all tests in the file (by initializing it in beforeAll rather than beforeEach) + beforeAll(async () => { + // This is for the benefit of the Azure Pipelines Hosted Windows agents, which come with + // webdrivers preinstalled but not on the PATH where Selenium looks for them by default. + // See https://docs.microsoft.com/en-us/azure/devops/pipelines/test/continuous-test-selenium#decide-how-you-will-deploy-and-test-your-app + if (process.env.ChromeWebDriver) { + const hostedAgentChromedriverPath = path.join(process.env.ChromeWebDriver, 'chromedriver.exe'); + const chromeService = new chrome.ServiceBuilder(hostedAgentChromedriverPath).build(); + chrome.setDefaultService(chromeService); + } + + // Selenium supports many browsers, not just Chrome. + // See https://www.npmjs.com/package/selenium-webdriver for examples. + driver = new Builder() + .forBrowser('chrome') + .setChromeOptions(new chrome.Options().headless()) + .build(); + }, TEST_TIMEOUT_MS); + + afterAll(async () => { + await driver.quit(); + }, TEST_TIMEOUT_MS); + + beforeEach(async () => { + // For simplicity, we're pointing our test browser directly to a static html file on disk. + // + // In a real project, you would probably use a localhost http server (Express.js, for example) + // and point selenium-webdriver to a http://localhost link. + // + // See https://jestjs.io/docs/en/testing-frameworks for examples. + const pageUnderTest = 'file://' + path.join(__dirname, '..', 'src', 'index.html'); + await driver.get(pageUnderTest); + + // Checking for a known element on the page in beforeEach serves two purposes: + // * It acts as a sanity check that our browser automation setup basically works + // * It ensures that the page is loaded before we run our accessibility scans + await driver.wait(until.elementLocated(By.css('h1'))); + }, TEST_TIMEOUT_MS); + + it('only contains known accessibility violations', async () => { + // Run an accessibility scan using axe-webdriverjs + const axeResults = await AxeBuilder(driver).analyze(); + + // Write a test expectation that accounts for "known" issues we want to baseline + expect(axeResults.violations.length).toBe(3); + + // Write the axe results to a .sarif file, so we can use the SARIF Multitool to + // apply a baseline file and show the results in the Scans tab in Azure Pipelines + const sarifResults = convertAxeToSarif(axeResults); + const testResultsDirectory = path.join(__dirname, '..', 'test-results'); + await promisify(fs.mkdir)(testResultsDirectory, { recursive: true }); + await promisify(fs.writeFile)( + path.join(testResultsDirectory, 'index.html.axe-core.sarif'), + // We'll be checking in the resulting .sarif file for baselining purposes, so + // it's a good idea to use a spacing argument (here, "2") to pretty-print the + // JSON. This makes it much more pleasant to diff when it changes. + JSON.stringify(sarifResults, null, 2)); + }, TEST_TIMEOUT_MS); +}); diff --git a/typescript-selenium-webdriver/yarn.lock b/typescript-selenium-webdriver/yarn.lock index 874276f7..1e79a534 100644 --- a/typescript-selenium-webdriver/yarn.lock +++ b/typescript-selenium-webdriver/yarn.lock @@ -1762,6 +1762,28 @@ jest-changed-files@^24.8.0: execa "^1.0.0" throat "^4.0.0" +jest-circus@^24.8.0: + version "24.8.0" + resolved "https://registry.yarnpkg.com/jest-circus/-/jest-circus-24.8.0.tgz#24ef638ac178fdf614a0d464e708d2b6e6900979" + integrity sha512-2QASG3QuDdk0SMP2O73D8u3/lc/A/E2G7q23v5WhbUR+hCGzWZXwRMKif18f11dSLfL1wcrMbwE4IorvV0DRVw== + dependencies: + "@babel/traverse" "^7.1.0" + "@jest/environment" "^24.8.0" + "@jest/test-result" "^24.8.0" + "@jest/types" "^24.8.0" + chalk "^2.0.1" + co "^4.6.0" + expect "^24.8.0" + is-generator-fn "^2.0.0" + jest-each "^24.8.0" + jest-matcher-utils "^24.8.0" + jest-message-util "^24.8.0" + jest-snapshot "^24.8.0" + jest-util "^24.8.0" + pretty-format "^24.8.0" + stack-utils "^1.0.1" + throat "^4.0.0" + jest-cli@^24.8.0: version "24.8.0" resolved "https://registry.yarnpkg.com/jest-cli/-/jest-cli-24.8.0.tgz#b075ac914492ed114fa338ade7362a301693e989"