Skip to content

Commit

Permalink
Several changes to runtests.py
Browse files Browse the repository at this point in the history 
Remove the flag-based test ignoring of -simdjs as it has
caused confusion on whether or not tests were running.
Add test tagging for simd, asmjs, and wasm functionality.
Add the ability to read --sanitize flags and exclude known
failing tests.

Edit: Allow all of the asmjs tests, since they work now.

@obastemur fixed the issues with them in chakra-core#2716, so I've gone and
enabled them here.

Edit: Add --static to test flags.

Not using it currently to mask out any tests; this is just to make
the test and build flags line up better.

Edit: Add a -j flag to runtests.py for num processes

This makes it line up better with the build flags for CI

Edit: Add a flag to warn instead of fail on test timeout

This is in an attempt to reduce noise a bit; a small number of tests
fail to run quickly sometimes enough on one channel of our test infra.
  • Loading branch information
@Penguinwizzard
Penguinwizzard committed Mar 28, 2017
1 parent 1e78b3a commit 8f3f4cd
Show file tree
Hide file tree
Showing 3 changed files with 79 additions and 41 deletions.
14 changes: 7 additions & 7 deletions test/Bugs/rlexe.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@
<default>
<files>SIMDExtendArgCSE.js</files>
<baseline>SIMDExtendArgCSE.baseline</baseline>
<tags>exclude_dynapogo,exclude_ship,require_backend</tags>
<tags>exclude_dynapogo,exclude_ship,require_backend,require_simd</tags>
<compile-flags>-bgjit- -on:asmjs -simdjs -testtrace:asmjs -AsmJsStopOnError</compile-flags>
</default>
</test>
Expand Down Expand Up @@ -280,7 +280,7 @@
<default>
<files>b208.js</files>
<baseline>b208.baseline</baseline>
<tags>require_backend</tags>
<tags>require_backend,require_simd</tags>
<compile-flags> -bgjit- -simdjs -simd128typespec -mic:1 -lic:1 -off:simplejit</compile-flags>
</default>
</test>
Expand All @@ -307,39 +307,39 @@
<default>
<files>b208_asmjs.js</files>
<baseline>b208_asmjs.baseline</baseline>
<tags>exclude_dynapogo,exclude_ship,require_backend</tags>
<tags>exclude_dynapogo,exclude_ship,require_backend,require_simd</tags>
<compile-flags>-simdjs -AsmJsStopOnError</compile-flags>
</default>
</test>
<test>
<default>
<files>b208_asmjs.js</files>
<baseline>b208_asmjs.baseline</baseline>
<tags>exclude_dynapogo,exclude_ship,require_backend</tags>
<tags>exclude_dynapogo,exclude_ship,require_backend,require_simd</tags>
<compile-flags> -bgjit- -simdjs -simd128typespec -asmjs- -mic:1 -lic:1 -off:simplejit</compile-flags>
</default>
</test>
<test>
<default>
<files>testSimdManyVars.js</files>
<baseline>testSimdManyVars.baseline</baseline>
<tags>exclude_dynapogo,exclude_ship,require_backend</tags>
<tags>exclude_dynapogo,exclude_ship,require_backend,require_simd</tags>
<compile-flags> -simdjs -asmjs -off:backend -testtrace:asmjs</compile-flags>
</default>
</test>
<test>
<default>
<files>b95.js</files>
<baseline>b95.baseline</baseline>
<tags>exclude_dynapogo,exclude_ship,require_backend</tags>
<tags>exclude_dynapogo,exclude_ship,require_backend,require_simd</tags>
<compile-flags>-simdjs -testtrace:asmjs -asmjs</compile-flags>
</default>
</test>
<test>
<default>
<files>b108.js</files>
<baseline>b108.baseline</baseline>
<tags>exclude_dynapogo,exclude_ship,require_backend</tags>
<tags>exclude_dynapogo,exclude_ship,require_backend,require_simd</tags>
<compile-flags>-simdjs -asmjs -testtrace:asmjs -asmjsstoponerror</compile-flags>
</default>
</test>
Expand Down
42 changes: 21 additions & 21 deletions test/rlexedirs.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -260,127 +260,127 @@
<dir>
<default>
<files>AsmJs</files>
<tags>exclude_serialized,exclude_arm,exclude_arm64,require_backend</tags>
<tags>exclude_serialized,exclude_arm,exclude_arm64,require_backend,require_asmjs</tags>
</default>
</dir>
<dir>
<default>
<files>AsmJsFloat</files>
<tags>exclude_serialized,exclude_arm,exclude_arm64,require_backend</tags>
<tags>exclude_serialized,exclude_arm,exclude_arm64,require_backend,require_asmjs</tags>
</default>
</dir>
<dir>
<default>
<files>AsmJsParser</files>
<tags>exclude_serialized,exclude_arm,exclude_arm64,require_backend</tags>
<tags>exclude_serialized,exclude_arm,exclude_arm64,require_backend,require_asmjs</tags>
</default>
</dir>
<dir>
<default>
<files>SIMD.float32x4</files>
<tags>exclude_arm,exclude_arm64,require_backend</tags>
<tags>exclude_arm,exclude_arm64,require_backend,require_simd</tags>
</default>
</dir>
<dir>
<default>
<files>SIMD.int32x4</files>
<tags>exclude_arm,exclude_arm64,require_backend</tags>
<tags>exclude_arm,exclude_arm64,require_backend,require_simd</tags>
</default>
</dir>
<dir>
<default>
<files>SIMD.int8x16</files>
<tags>exclude_arm,exclude_arm64,require_backend</tags>
<tags>exclude_arm,exclude_arm64,require_backend,require_simd</tags>
</default>
</dir>
<dir>
<default>
<files>SIMD.float32x4.asmjs</files>
<tags>exclude_arm,exclude_arm64,require_backend</tags>
<tags>exclude_arm,exclude_arm64,require_backend,require_simd</tags>
</default>
</dir>
<dir>
<default>
<files>SIMD.int32x4.asmjs</files>
<tags>exclude_arm,exclude_arm64,require_backend</tags>
<tags>exclude_arm,exclude_arm64,require_backend,require_simd</tags>
</default>
</dir>
<dir>
<default>
<files>SIMD.int16x8.asmjs</files>
<tags>exclude_arm,exclude_arm64,require_backend</tags>
<tags>exclude_arm,exclude_arm64,require_backend,require_simd</tags>
</default>
</dir>
<dir>
<default>
<files>SIMD.int8x16.asmjs</files>
<tags>exclude_arm,exclude_arm64,require_backend</tags>
<tags>exclude_arm,exclude_arm64,require_backend,require_simd</tags>
</default>
</dir>
<dir>
<default>
<files>SIMD.uint32x4.asmjs</files>
<tags>exclude_arm,exclude_arm64,require_backend</tags>
<tags>exclude_arm,exclude_arm64,require_backend,require_simd</tags>
</default>
</dir>
<dir>
<default>
<files>SIMD.uint16x8.asmjs</files>
<tags>exclude_arm,exclude_arm64,require_backend</tags>
<tags>exclude_arm,exclude_arm64,require_backend,require_simd</tags>
</default>
</dir>
<dir>
<default>
<files>SIMD.uint8x16.asmjs</files>
<tags>exclude_arm,exclude_arm64,require_backend</tags>
<tags>exclude_arm,exclude_arm64,require_backend,require_simd</tags>
</default>
</dir>
<dir>
<default>
<files>SIMD.bool32x4.asmjs</files>
<tags>exclude_arm,exclude_arm64,require_backend</tags>
<tags>exclude_arm,exclude_arm64,require_backend,require_simd</tags>
</default>
</dir>
<dir>
<default>
<files>SIMD.bool16x8.asmjs</files>
<tags>exclude_arm,exclude_arm64,require_backend</tags>
<tags>exclude_arm,exclude_arm64,require_backend,require_simd</tags>
</default>
</dir>
<dir>
<default>
<files>SIMD.bool8x16.asmjs</files>
<tags>exclude_arm,exclude_arm64,require_backend</tags>
<tags>exclude_arm,exclude_arm64,require_backend,require_simd</tags>
</default>
</dir>
<dir>
<default>
<files>SIMD.workloads.asmjs</files>
<tags>exclude_serialized,exclude_arm,exclude_arm64,require_backend</tags>
<tags>exclude_serialized,exclude_arm,exclude_arm64,require_backend,require_simd</tags>
</default>
</dir>
<dir>
<default>
<files>SIMD.wrappers</files>
<tags>exclude_serialized,exclude_arm,exclude_arm64,require_backend</tags>
<tags>exclude_serialized,exclude_arm,exclude_arm64,require_backend,require_simd</tags>
</default>
</dir>
<dir>
<default>
<files>SIMD.TypeSpec</files>
<tags>exclude_serialized,exclude_arm,exclude_arm64,require_backend</tags>
<tags>exclude_serialized,exclude_arm,exclude_arm64,require_backend,require_simd</tags>
</default>
</dir>
<dir>
<default>
<files>WasmSpec</files>
<tags>exclude_serialized,exclude_arm,exclude_arm64,require_backend</tags>
<tags>exclude_serialized,exclude_arm,exclude_arm64,require_backend,require_wasm</tags>
</default>
</dir>
<dir>
<default>
<files>wasm</files>
<tags>exclude_serialized,exclude_arm,exclude_arm64,require_backend</tags>
<tags>exclude_serialized,exclude_arm,exclude_arm64,require_backend,require_wasm</tags>
<tags>exclude_arm,exclude_arm64</tags>
</default>
</dir>
Expand Down
64 changes: 51 additions & 13 deletions test/runtests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

from __future__ import print_function
from datetime import datetime
from multiprocessing import Pool, Manager
from multiprocessing import Pool, Manager, cpu_count
from threading import Timer
import sys
import os
Expand Down Expand Up @@ -40,10 +40,18 @@

parser.add_argument('folders', metavar='folder', nargs='*',
help='folder subset to run tests')
parser.add_argument('-b', '--binary', metavar='bin', help='ch full path')
parser.add_argument('-b', '--binary', metavar='bin',
help='ch full path')
parser.add_argument('-v',
help='unused') # to mirror build.sh arguments
parser.add_argument('--sanitize', metavar='sanitizers',
help='ignore tests known to be broken with these sanitizers')
parser.add_argument('-d', '--debug', action='store_true',
help='use debug build');
parser.add_argument('-t', '--test', action='store_true', help='use test build')
parser.add_argument('-t', '--test', '--test-build', action='store_true',
help='use test build')
parser.add_argument('--static', action='store_true',
help='mark that we are testing a static build')
parser.add_argument('--variants', metavar='variant', nargs='+',
help='run specified test variants')
parser.add_argument('--include-slow', action='store_true',
Expand All @@ -60,13 +68,20 @@
help='global test flags to ch')
parser.add_argument('--timeout', type=int, default=DEFAULT_TIMEOUT,
help='test timeout (default ' + str(DEFAULT_TIMEOUT) + ' seconds)')
parser.add_argument('--swb', action='store_true', help='use binary from VcBuild.SWB to run the test')
parser.add_argument('-l', '--logfile', metavar='logfile', help='file to log results to', default=None)
parser.add_argument('--x86', action='store_true', help='use x86 build')
parser.add_argument('--x64', action='store_true', help='use x64 build')
parser.add_argument('--swb', action='store_true',
help='use binary from VcBuild.SWB to run the test')
parser.add_argument('-l', '--logfile', metavar='logfile',
help='file to log results to', default=None)
parser.add_argument('--x86', action='store_true',
help='use x86 build')
parser.add_argument('--x64', action='store_true',
help='use x64 build')
parser.add_argument('-j', '--processcount', metavar='processcount',
help='number of parallel threads to use')
parser.add_argument('--warn-on-timeout', action='store_true',
help='warn when a test times out instead of labelling it as an error immediately')
args = parser.parse_args()


test_root = os.path.dirname(os.path.realpath(__file__))
repo_root = os.path.dirname(test_root)

Expand Down Expand Up @@ -125,11 +140,28 @@
if sys.platform != 'win32':
not_tags.add('exclude_xplat')
not_tags.add('Intl')
not_tags.add('require_simd')
not_tags.add('require_wasm')

if args.sanitize != None:
not_tags.add('exclude_sanitize_'+args.sanitize)

if args.static != None:
not_tags.add('exclude_static')

if sys.platform == 'darwin':
not_tags.add('exclude_mac')
not_compile_flags = set(['-simdjs']) \
if sys.platform != 'win32' else None
not_compile_flags = None

# use -j flag to specify number of parallel processes
processcount = cpu_count()
if args.processcount != None:
processcount = int(args.processcount)

# handle warn on timeout
warn_on_timeout = False
if args.warn_on_timeout == True:
warn_on_timeout = True

# use tags/not_tags/not_compile_flags as case-insensitive
def lower_set(s):
Expand Down Expand Up @@ -335,7 +367,10 @@ def _process_one_msg(self):
def _show_failed(self, test, flags, exit_code, output,
expected_output=None, timedout=False):
if timedout:
self._print('ERROR: Test timed out!')
if warn_on_timeout:
self._print('WARNING: Test timed out!')
else:
self._print('ERROR: Test timed out!')
self._print('{} {} {}'.format(binary, ' '.join(flags), test.filename))
if expected_output == None or timedout:
self._print("\nOutput:")
Expand All @@ -359,7 +394,10 @@ def _show_failed(self, test, flags, exit_code, output,
break

self._print("exit code: {}".format(exit_code))
self._log_result(test, fail=True)
if warn_on_timeout:
self._log_result(test, fail=False)
else:
self._log_result(test, fail=True)

# temp: try find real file name on hard drive if case mismatch
def _check_file(self, folder, filename):
Expand Down Expand Up @@ -646,7 +684,7 @@ def main():
os.remove(f)

# run each variant
pool, sequential_pool = Pool(), Pool(1)
pool, sequential_pool = Pool(processcount), Pool(1)
start_time = datetime.now()
for variant in variants:
variant.run(testLoader, pool, sequential_pool)
Expand Down

0 comments on commit 8f3f4cd

Please sign in to comment.