From dfcb8b487df661bb674159d893c4cfa3bee156a2 Mon Sep 17 00:00:00 2001
From: "Alexander V. Tikhonov" <avtikhon@tarantool.org>
Date: Wed, 8 Jul 2020 14:34:46 +0300
Subject: [PATCH] Enable test reruns on failed fragiled tests

Added ability to set per suite in suite.ini configuration file
'retries' option, which sets the number of accepted reruns of
the tests failed from 'fragile' list:

  fragile = {
    "retries": 10,
    "tests": {
        "bitset.test.lua": {
            "issues": [ "gh-4095" ],
        }
    }}

Part of #189
---
 lib/test_suite.py |  5 ++++-
 lib/worker.py     | 15 ++++++++++++++-
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/lib/test_suite.py b/lib/test_suite.py
index d37b674d..b8e1454c 100644
--- a/lib/test_suite.py
+++ b/lib/test_suite.py
@@ -81,7 +81,7 @@ def __init__(self, suite_path, args):
         self.args = args
         self.tests = []
         self.ini = {}
-        self.fragile = {'tests': {}}
+        self.fragile = {'retries': 0, 'tests': {}}
         self.suite_path = suite_path
         self.ini["core"] = "tarantool"
 
@@ -265,5 +265,8 @@ def run_test(self, test, server, inspector):
     def is_parallel(self):
         return self.ini['is_parallel']
 
+    def fragile_retries(self):
+        return self.fragile.get('retries', 0)
+
     def show_reproduce_content(self):
         return self.ini['show_reproduce_content']
diff --git a/lib/worker.py b/lib/worker.py
index de3ee563..1f70abb7 100644
--- a/lib/worker.py
+++ b/lib/worker.py
@@ -326,8 +326,21 @@ def run_loop(self, task_queue, result_queue):
                 self.stop_worker(task_queue, result_queue)
                 break
 
+            short_status = None
             result_queue.put(self.current_task(task_id))
-            short_status = self.run_task(task_id)
+            retries_left = self.suite.fragile_retries()
+            # let's run till short_status became 'pass'
+            while short_status != 'pass' and retries_left >= 0:
+                # print message only after some fails occurred
+                if short_status == 'fail':
+                    color_stdout(
+                        'Test "%s", conf: "%s"\n'
+                        '\tfrom "fragile" list failed, rerunning ...\n'
+                        % (task_id[0], task_id[1]), schema='error')
+                # run task and save the result to short_status
+                short_status = self.run_task(task_id)
+                retries_left = retries_left - 1
+
             result_queue.put(self.wrap_result(task_id, short_status))
             if not lib.Options().args.is_force and short_status == 'fail':
                 color_stdout(