Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

use jest-circus runner to improve beforeAll/beforeEach failure behavior #12

Merged
merged 4 commits into from
May 23, 2019
Merged

use jest-circus runner to improve beforeAll/beforeEach failure behavior #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
76461da
use jest-circus runner to improve beforeAll/beforeEach failure behavior
dbjorge May 22, 2019
209dcb9
Update timeout syntax to be compatible with jest-circus
dbjorge May 22, 2019
1b2120a
Merge branch 'master' into use-jest-circus
dbjorge May 22, 2019
0df43d0
Merge branch 'master' into use-jest-circus
dbjorge May 23, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions typescript-selenium-webdriver/jest.config.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,18 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
module.exports = {
// These are necessary to use typescript with Jest tests
preset:'ts-jest',
testEnvironment: 'node',

// Using the jest-circus testRunner ensures that tests will not run if beforeAll or beforeEach fails.
//
// Since browser automation tests commonly use beforeAll/beforeEach to perform browser/page setup,
// it's usually misleading to allow tests to continue trying to execute when they have failed.
//
// See https://github.com/facebook/jest/issues/2713 for details.
testRunner: 'jest-circus/runner',

reporters: [
// This enables console output for local development and build log output in Pipelines.
'default',
Expand Down
1 change: 1 addition & 0 deletions typescript-selenium-webdriver/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
"axe-webdriverjs": "^2.2.0",
"http-server": "^0.11.1",
"jest": "^24.8.0",
"jest-circus": "^24.8.0",
"jest-junit": "^6.4.0",
"selenium-webdriver": "^4.0.0-alpha.1",
"ts-jest": "^24.0.2",
Expand Down
152 changes: 76 additions & 76 deletions typescript-selenium-webdriver/tests/index.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,76 +1,76 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
import { convertAxeToSarif } from 'axe-sarif-converter';
import * as AxeBuilder from 'axe-webdriverjs';
import * as fs from 'fs';
import * as path from 'path';
import { Builder, By, ThenableWebDriver, until } from 'selenium-webdriver';
import * as chrome from 'selenium-webdriver/chrome';
import { promisify } from 'util';
describe('index.html', () => {
let driver: ThenableWebDriver;
// Starting a browser instance is time-consuming, so we share one browser instance between
// all tests in the file (by initializing it in beforeAll rather than beforeEach)
beforeAll(async () => {
// The default timeout (5 seconds) is not always enough to start/quit a browser instance
jest.setTimeout(30000);
// This is for the benefit of the Azure Pipelines Hosted Windows agents, which come with
// webdrivers preinstalled but not on the PATH where Selenium looks for them by default.
// See https://docs.microsoft.com/en-us/azure/devops/pipelines/test/continuous-test-selenium#decide-how-you-will-deploy-and-test-your-app
if (process.env.ChromeWebDriver) {
const hostedAgentChromedriverPath = path.join(process.env.ChromeWebDriver, 'chromedriver.exe');
const chromeService = new chrome.ServiceBuilder(hostedAgentChromedriverPath).build();
chrome.setDefaultService(chromeService);
}
// Selenium supports many browsers, not just Chrome.
// See https://www.npmjs.com/package/selenium-webdriver for examples.
driver = new Builder()
.forBrowser('chrome')
.setChromeOptions(new chrome.Options().headless())
.build();
});
afterAll(async () => {
await driver.quit();
});
beforeEach(async () => {
// For simplicity, we're pointing our test browser directly to a static html file on disk.
//
// In a real project, you would probably use a localhost http server (Express.js, for example)
// and point selenium-webdriver to a http://localhost link.
//
// See https://jestjs.io/docs/en/testing-frameworks for examples.
const pageUnderTest = 'file://' + path.join(__dirname, '..', 'src', 'index.html');
await driver.get(pageUnderTest);
// Checking for a known element on the page in beforeEach serves two purposes:
// * It acts as a sanity check that our browser automation setup basically works
// * It ensures that the page is loaded before we run our accessibility scans
await driver.wait(until.elementLocated(By.css('h1')));
});
it('only contains known accessibility violations', async () => {
// Run an accessibility scan using axe-webdriverjs
const axeResults = await AxeBuilder(driver).analyze();
// Write a test expectation that accounts for "known" issues we want to baseline
expect(axeResults.violations.length).toBe(3);
// Write the axe results to a .sarif file, so we can use the SARIF Multitool to
// apply a baseline file and show the results in the Scans tab in Azure Pipelines
const sarifResults = convertAxeToSarif(axeResults);
const testResultsDirectory = path.join(__dirname, '..', 'test-results');
await promisify(fs.mkdir)(testResultsDirectory, { recursive: true });
await promisify(fs.writeFile)(
path.join(testResultsDirectory, 'index.html.axe-core.sarif'),
// We'll be checking in the resulting .sarif file for baselining purposes, so
// it's a good idea to use a spacing argument (here, "2") to pretty-print the
// JSON. This makes it much more pleasant to diff when it changes.
JSON.stringify(sarifResults, null, 2));
});
});
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
import { convertAxeToSarif } from 'axe-sarif-converter';
import * as AxeBuilder from 'axe-webdriverjs';
import * as fs from 'fs';
import * as path from 'path';
import { Builder, By, ThenableWebDriver, until } from 'selenium-webdriver';
import * as chrome from 'selenium-webdriver/chrome';
import { promisify } from 'util';

// The default timeout for tests/fixtures (5 seconds) is not always enough to start/quit/navigate a browser instance.
const TEST_TIMEOUT_MS = 30000;
dbjorge marked this conversation as resolved.
Show resolved Hide resolved

describe('index.html', () => {
let driver: ThenableWebDriver;

// Starting a browser instance is time-consuming, so we share one browser instance between
// all tests in the file (by initializing it in beforeAll rather than beforeEach)
beforeAll(async () => {
// This is for the benefit of the Azure Pipelines Hosted Windows agents, which come with
// webdrivers preinstalled but not on the PATH where Selenium looks for them by default.
// See https://docs.microsoft.com/en-us/azure/devops/pipelines/test/continuous-test-selenium#decide-how-you-will-deploy-and-test-your-app
if (process.env.ChromeWebDriver) {
const hostedAgentChromedriverPath = path.join(process.env.ChromeWebDriver, 'chromedriver.exe');
const chromeService = new chrome.ServiceBuilder(hostedAgentChromedriverPath).build();
chrome.setDefaultService(chromeService);
}

// Selenium supports many browsers, not just Chrome.
// See https://www.npmjs.com/package/selenium-webdriver for examples.
driver = new Builder()
.forBrowser('chrome')
.setChromeOptions(new chrome.Options().headless())
.build();
}, TEST_TIMEOUT_MS);

afterAll(async () => {
await driver.quit();
}, TEST_TIMEOUT_MS);

beforeEach(async () => {
// For simplicity, we're pointing our test browser directly to a static html file on disk.
//
// In a real project, you would probably use a localhost http server (Express.js, for example)
// and point selenium-webdriver to a http://localhost link.
//
// See https://jestjs.io/docs/en/testing-frameworks for examples.
const pageUnderTest = 'file://' + path.join(__dirname, '..', 'src', 'index.html');
await driver.get(pageUnderTest);

// Checking for a known element on the page in beforeEach serves two purposes:
// * It acts as a sanity check that our browser automation setup basically works
// * It ensures that the page is loaded before we run our accessibility scans
await driver.wait(until.elementLocated(By.css('h1')));
}, TEST_TIMEOUT_MS);

it('only contains known accessibility violations', async () => {
// Run an accessibility scan using axe-webdriverjs
const axeResults = await AxeBuilder(driver).analyze();

// Write a test expectation that accounts for "known" issues we want to baseline
expect(axeResults.violations.length).toBe(3);

// Write the axe results to a .sarif file, so we can use the SARIF Multitool to
// apply a baseline file and show the results in the Scans tab in Azure Pipelines
const sarifResults = convertAxeToSarif(axeResults);
const testResultsDirectory = path.join(__dirname, '..', 'test-results');
await promisify(fs.mkdir)(testResultsDirectory, { recursive: true });
await promisify(fs.writeFile)(
path.join(testResultsDirectory, 'index.html.axe-core.sarif'),
// We'll be checking in the resulting .sarif file for baselining purposes, so
// it's a good idea to use a spacing argument (here, "2") to pretty-print the
// JSON. This makes it much more pleasant to diff when it changes.
JSON.stringify(sarifResults, null, 2));
}, TEST_TIMEOUT_MS);
});
22 changes: 22 additions & 0 deletions typescript-selenium-webdriver/yarn.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1762,6 +1762,28 @@ jest-changed-files@^24.8.0:
execa "^1.0.0"
throat "^4.0.0"

jest-circus@^24.8.0:
version "24.8.0"
resolved "https://registry.yarnpkg.com/jest-circus/-/jest-circus-24.8.0.tgz#24ef638ac178fdf614a0d464e708d2b6e6900979"
integrity sha512-2QASG3QuDdk0SMP2O73D8u3/lc/A/E2G7q23v5WhbUR+hCGzWZXwRMKif18f11dSLfL1wcrMbwE4IorvV0DRVw==
dependencies:
"@babel/traverse" "^7.1.0"
"@jest/environment" "^24.8.0"
"@jest/test-result" "^24.8.0"
"@jest/types" "^24.8.0"
chalk "^2.0.1"
co "^4.6.0"
expect "^24.8.0"
is-generator-fn "^2.0.0"
jest-each "^24.8.0"
jest-matcher-utils "^24.8.0"
jest-message-util "^24.8.0"
jest-snapshot "^24.8.0"
jest-util "^24.8.0"
pretty-format "^24.8.0"
stack-utils "^1.0.1"
throat "^4.0.0"

jest-cli@^24.8.0:
version "24.8.0"
resolved "https://registry.yarnpkg.com/jest-cli/-/jest-cli-24.8.0.tgz#b075ac914492ed114fa338ade7362a301693e989"
Expand Down