Add test_timeout to limit test run time

lib/app_server.py

@@ -6,7 +6,9 @@
 
 from gevent.subprocess import Popen, PIPE
 
+from lib.colorer import color_stdout
 from lib.colorer import color_log
+from lib.options import Options
 from lib.preprocessor import TestState
 from lib.server import Server
 from lib.server import DEFAULT_SNAPSHOT_NAME
@@ -17,12 +19,22 @@
 from lib.utils import format_process
 from lib.utils import warn_unix_socket
 from test import TestRunGreenlet, TestExecutionError
+from threading import Timer
+
+
+def timeout_handler(server_process, test_timeout):
+    color_stdout("Test timeout of %d secs reached\t" % test_timeout, schema='error')
+    server_process.kill()
 
 
 def run_server(execs, cwd, server, logfile, retval):
     os.putenv("LISTEN", server.iproto)
     server.process = Popen(execs, stdout=PIPE, stderr=PIPE, cwd=cwd)
+    test_timeout = Options().args.test_timeout
+    timer = Timer(test_timeout, timeout_handler, (server.process, test_timeout))
+    timer.start()
     stdout, stderr = server.process.communicate()
+    timer.cancel()
     sys.stdout.write(stdout)
     with open(logfile, 'a') as f:
         f.write(stderr)

lib/options.py

@@ -190,6 +190,14 @@ def __init__(self):
                 Such files created by workers in the "var/reproduce" directory.
                 Note: The option works now only with parallel testing.""")
 
+        parser.add_argument(
+                "--test-timeout",
+                dest="test_timeout",
+                default=110,
+                type=int,
+                help="""Break the test process with kill signal if the test runs
+                longer than this amount of seconds. Default: 110 [seconds].""")
+
         parser.add_argument(
                 "--no-output-timeout",
                 dest="no_output_timeout",

lib/tarantool_server.py

@@ -14,6 +14,7 @@
 import yaml
 
 from gevent import socket
+from gevent import Timeout
 from greenlet import GreenletExit
 from threading import Timer
 
@@ -27,6 +28,7 @@
 from lib.colorer import color_stdout
 from lib.colorer import color_log
 from lib.colorer import qa_notice
+from lib.options import Options
 from lib.preprocessor import TestState
 from lib.server import Server
 from lib.server import DEFAULT_SNAPSHOT_NAME
@@ -44,12 +46,21 @@
 def save_join(green_obj, timeout=None):
     """
     Gevent join wrapper for
-    test-run stop-on-crash feature
+    test-run stop-on-crash/stop-on-timeout feature
 
-    :return True in case of crash and False otherwise
+    :return True in case of crash or test timeout and False otherwise
     """
     try:
-        green_obj.join(timeout=timeout)
+        green_obj.get(timeout=timeout)
+    except Timeout:
+        color_stdout("Test timeout of %d secs reached\t" % timeout, schema='error')
+        # We should kill the greenlet that writes to a temporary
+        # result file. If the same test is run several times (e.g.
+        # on different configurations), this greenlet may wake up
+        # and write to the temporary result file of the new run of
+        # the test.
+        green_obj.kill()
+        return True
     except GreenletExit:
         return True
     # We don't catch TarantoolStartError here to propagate it to a parent
@@ -60,7 +71,6 @@ def save_join(green_obj, timeout=None):
 class LuaTest(Test):
     """ Handle *.test.lua and *.test.sql test files. """
 
-    TIMEOUT = 60 * 10
     RESULT_FILE_VERSION_INITIAL = 1
     RESULT_FILE_VERSION_DEFAULT = 2
     RESULT_FILE_VERSION_LINE_RE = re.compile(
@@ -378,7 +388,7 @@ def execute(self, server):
         lua.start()
         crash_occured = True
         try:
-            crash_occured = save_join(lua, timeout=self.TIMEOUT)
+            crash_occured = save_join(lua, timeout=Options().args.test_timeout)
             self.killall_servers(server, ts, crash_occured)
         except KeyboardInterrupt:
             # prevent tests greenlet from writing to the real stdout