diff --git a/lib/app_server.py b/lib/app_server.py index a046b21a..a02200b6 100644 --- a/lib/app_server.py +++ b/lib/app_server.py @@ -6,7 +6,9 @@ from gevent.subprocess import Popen, PIPE +from lib.colorer import color_stdout from lib.colorer import color_log +from lib.options import Options from lib.preprocessor import TestState from lib.server import Server from lib.server import DEFAULT_SNAPSHOT_NAME @@ -17,12 +19,22 @@ from lib.utils import format_process from lib.utils import warn_unix_socket from test import TestRunGreenlet, TestExecutionError +from threading import Timer + + +def timeout_handler(server_process, test_timeout): + color_stdout("Test timeout of %d secs reached\t" % test_timeout, schema='error') + server_process.kill() def run_server(execs, cwd, server, logfile, retval): os.putenv("LISTEN", server.iproto) server.process = Popen(execs, stdout=PIPE, stderr=PIPE, cwd=cwd) + test_timeout = Options().args.test_timeout + timer = Timer(test_timeout, timeout_handler, (server.process, test_timeout)) + timer.start() stdout, stderr = server.process.communicate() + timer.cancel() sys.stdout.write(stdout) with open(logfile, 'a') as f: f.write(stderr) diff --git a/lib/options.py b/lib/options.py index 9c81a9a1..5506b3a1 100644 --- a/lib/options.py +++ b/lib/options.py @@ -190,6 +190,14 @@ def __init__(self): Such files created by workers in the "var/reproduce" directory. Note: The option works now only with parallel testing.""") + parser.add_argument( + "--test-timeout", + dest="test_timeout", + default=110, + type=int, + help="""Break the test process with kill signal if the test runs + longer than this amount of seconds. Default: 110 [seconds].""") + parser.add_argument( "--no-output-timeout", dest="no_output_timeout", diff --git a/lib/tarantool_server.py b/lib/tarantool_server.py index 481b08f9..bdbc9358 100644 --- a/lib/tarantool_server.py +++ b/lib/tarantool_server.py @@ -14,6 +14,7 @@ import yaml from gevent import socket +from gevent import Timeout from greenlet import GreenletExit from threading import Timer @@ -27,6 +28,7 @@ from lib.colorer import color_stdout from lib.colorer import color_log from lib.colorer import qa_notice +from lib.options import Options from lib.preprocessor import TestState from lib.server import Server from lib.server import DEFAULT_SNAPSHOT_NAME @@ -44,12 +46,21 @@ def save_join(green_obj, timeout=None): """ Gevent join wrapper for - test-run stop-on-crash feature + test-run stop-on-crash/stop-on-timeout feature - :return True in case of crash and False otherwise + :return True in case of crash or test timeout and False otherwise """ try: - green_obj.join(timeout=timeout) + green_obj.get(timeout=timeout) + except Timeout: + color_stdout("Test timeout of %d secs reached\t" % timeout, schema='error') + # We should kill the greenlet that writes to a temporary + # result file. If the same test is run several times (e.g. + # on different configurations), this greenlet may wake up + # and write to the temporary result file of the new run of + # the test. + green_obj.kill() + return True except GreenletExit: return True # We don't catch TarantoolStartError here to propagate it to a parent @@ -60,7 +71,6 @@ def save_join(green_obj, timeout=None): class LuaTest(Test): """ Handle *.test.lua and *.test.sql test files. """ - TIMEOUT = 60 * 10 RESULT_FILE_VERSION_INITIAL = 1 RESULT_FILE_VERSION_DEFAULT = 2 RESULT_FILE_VERSION_LINE_RE = re.compile( @@ -378,7 +388,7 @@ def execute(self, server): lua.start() crash_occured = True try: - crash_occured = save_join(lua, timeout=self.TIMEOUT) + crash_occured = save_join(lua, timeout=Options().args.test_timeout) self.killall_servers(server, ts, crash_occured) except KeyboardInterrupt: # prevent tests greenlet from writing to the real stdout