Assert: Introduce timeout to set per-test timeout durations

Gruntfile.js

@@ -165,6 +165,7 @@ module.exports = function( grunt ) {
 				"test/main/test",
 				"test/main/assert",
 				"test/main/assert/step",
+				"test/main/assert/timeout",
 				"test/main/async",
 				"test/main/promise",
 				"test/main/modules",

docs/assert/timeout.md

@@ -0,0 +1,46 @@
+---
+layout: default
+title: timeout
+description: Sets the length of time to wait for async operations before failing the test.
+categories:
+  - assert
+  - async
+---
+
+## `timeout( duration )`
+
+Sets the length of time to wait for async operations before failing the test.
+
+| name | description |
+|------|-------------|
+| `duration` (Number) | The length of time, in milliseconds, to wait for async operations. |
+
+### Description
+
+`assert.timeout()` sets the length of time, in milliseconds, to wait for async operations in the current test. This is equivalent to setting `config.testTimeout` on a per-test basis. The timeout length only applies when performing async operations.
+
+If `0` is passed, then the test will be assumed to be completely synchronous. If a non-numeric value is passed as an argument, the function will throw an error.
+
+### Examples
+
+```js
+QUnit.test( "Waiting for focus event", function( assert ) {
+  assert.timeout( 1000 ); // Timeout of 1 second
+  var done = assert.async();
+  var input = $( "#test-input" ).focus();
+  setTimeout(function() {
+    assert.equal( document.activeElement, input[0], "Input was focused" );
+    done();
+  });
+});
+```
+
+```js
+QUnit.test( "Waiting for async function", function( assert ) {
+  assert.timeout( 500 ); // Timeout of .5 seconds
+  var promise = asyncFunction();
+  return promise.then( function( result ) {
+    assert.ok( result );
+  } );
+});
+```

src/assert.js

@@ -14,6 +14,14 @@ class Assert {
 
 	// Assert helpers
 
+	timeout( duration ) {
+		if ( typeof duration !== "number" ) {
+			throw new Error( "You must pass a number as the duration to assert.timeout" );
+		}
+
+		this.test.timeout = duration;
+	}
+
 	// Documents a "step", which is a string value, in a test as a passing assertion
 	step( message ) {
 		const result = !!message;

src/test.js

@@ -35,6 +35,7 @@ export default function Test( settings ) {
 	this.module = config.currentModule;
 	this.stack = sourceFromStacktrace( 3 );
 	this.steps = [];
+	this.timeout = undefined;
 
 	// If a module is skipped, all its tests and the tests of the child suites
 	// should be treated as skipped even if they are defined as `only` or `todo`.
@@ -164,6 +165,15 @@ Test.prototype = {
 		function runTest( test ) {
 			promise = test.callback.call( test.testEnvironment, test.assert );
 			test.resolvePromise( promise );
+
+			// If the test has a "lock" on it, but the timeout is 0, then we push a
+			// failure as the test should be synchronous.
+			if ( test.timeout === 0 && test.semaphore !== 0 ) {
+				pushFailure(
+					"Test did not finish synchronously even though assert.timeout( 0 ) was used.",
+					sourceFromStacktrace( 2 )
+				);
+			}
 		}
 	},
 
@@ -677,20 +687,33 @@ export function only( testName, callback ) {
 
 // Put a hold on processing and return a function that will release it.
 export function internalStop( test ) {
-	var released = false;
-
 	test.semaphore += 1;
 	config.blocking = true;
 
 	// Set a recovery timeout, if so configured.
-	if ( config.testTimeout && defined.setTimeout ) {
-		clearTimeout( config.timeout );
-		config.timeout = setTimeout( function() {
-			pushFailure( "Test timed out", sourceFromStacktrace( 2 ) );
-			internalRecover( test );
-		}, config.testTimeout );
+	if ( defined.setTimeout ) {
+		let timeoutDuration;
+
+		if ( typeof test.timeout === "number" ) {
+			timeoutDuration = test.timeout;
+		} else if ( typeof config.testTimeout === "number" ) {
+			timeoutDuration = config.testTimeout;
+		}
+
+		if ( typeof timeoutDuration === "number" && timeoutDuration > 0 ) {
+			clearTimeout( config.timeout );
+			config.timeout = setTimeout( function() {
+				pushFailure(
+					`Test took longer than ${timeoutDuration}ms; test timed out.`,
+					sourceFromStacktrace( 2 )
+				);
+				internalRecover( test );
+			}, timeoutDuration );
+		}
+
 	}
 
+	let released = false;
 	return function resume() {
 		if ( released ) {
 			return;

test/.eslintrc.json

@@ -13,5 +13,8 @@
 	"globals": {
 		"QUnit": false,
 		"console": false
+	},
+	"rules": {
+		"max-len": "off"
 	}
 }

test/index.html

@@ -8,6 +8,7 @@
 	<script src="main/test.js"></script>
 	<script src="main/assert.js"></script>
 	<script src="main/assert/step.js"></script>
+	<script src="main/assert/timeout.js"></script>
 	<script src="main/async.js"></script>
 	<script src="main/promise.js"></script>
 	<script src="main/dump.js"></script>

test/main/assert/timeout.js

@@ -0,0 +1,101 @@
+QUnit.module( "assert.timeout", function() {
+	QUnit.test( "pushes a failure if test times out when using async", function( assert ) {
+		assert.timeout( 10 );
+		assert.expect( 1 );
+
+		var originalPushFailure = QUnit.config.current.pushFailure;
+		QUnit.config.current.pushFailure = function pushFailureStub( message ) {
+			QUnit.config.current.pushFailure = originalPushFailure;
+
+			assert.equal( message, "Test took longer than 10ms; test timed out." );
+		};
+
+		assert.async();
+	} );
+
+	QUnit.test( "pushes a failure if test times out when using a promise", function( assert ) {
+		assert.timeout( 10 );
+		assert.expect( 1 );
+
+		var originalPushFailure = QUnit.config.current.pushFailure;
+		QUnit.config.current.pushFailure = function pushFailureStub( message ) {
+			QUnit.config.current.pushFailure = originalPushFailure;
+
+			assert.equal( message, "Test took longer than 10ms; test timed out." );
+		};
+
+		// Return a "thenable" to serve as a mock Promise
+		return {
+			then: function() {}
+		};
+	} );
+
+	QUnit.test( "does not push a failure if test is synchronous", function( assert ) {
+		assert.timeout( 1 );
+
+		var wait = Date.now() + 10;
+		while ( Date.now() < wait ) {}
+
+		assert.ok( true );
+	} );
+
+	QUnit.test( "throws an error if a non-numeric value is passed as duration", function( assert ) {
+		assert.throws( function() {
+			assert.timeout( null );
+		}, /You must pass a number as the duration to assert.timeout/ );
+	} );
+
+	QUnit.module( "a value of zero", function() {
+		function stubPushFailure( assert ) {
+			var originalPushFailure = QUnit.config.current.pushFailure;
+			QUnit.config.current.pushFailure = function pushFailureStub( message ) {
+				QUnit.config.current.pushFailure = originalPushFailure;
+
+				assert.equal(
+					message,
+					"Test did not finish synchronously even though assert.timeout( 0 ) was used."
+				);
+			};
+		}
+
+		QUnit.test( "does not fail a synchronous test using assert.async", function( assert ) {
+			assert.timeout( 0 );
+			var done = assert.async();
+			assert.ok( true );
+			done();
+		} );
+
+		QUnit.test( "fails a test using assert.async and a setTimeout of 0", function( assert ) {
+			assert.timeout( 0 );
+			assert.expect( 1 );
+
+			stubPushFailure( assert );
+
+			var done = assert.async();
+			setTimeout( done, 0 );
+		} );
+
+		if ( typeof Promise !== "undefined" ) {
+			QUnit.test( "fails a test returning an immediately resolved Promise", function( assert ) {
+				assert.timeout( 0 );
+				assert.expect( 1 );
+
+				stubPushFailure( assert );
+
+				return Promise.resolve();
+			} );
+
+			QUnit.test( "fails a test using assert.async and an immediately resolved Promise", function( assert ) {
+				assert.timeout( 0 );
+				assert.expect( 1 );
+
+				stubPushFailure( assert );
+
+				var done = assert.async();
+				Promise.resolve().then( done );
+			} );
+		}
+	} );
+} );
+
+