qt-testrunner: do not try to re-run initTestCase and others

Some function names are special for qtestlib, in the sense that they can
not be specified as a command line argument to run individually.

In such cases qt-testrunner treats the failure specially and tries once
to re-run the full test executable.

Fixes: QTBUG-89011
Change-Id: I0cc25f91c57374e5ac65ade10e2e223fe969f211
Reviewed-by: Volker Hilsheimer <[email protected]>
Reviewed-by: Daniel Smith <[email protected]>

1 files changed, 36 insertions, 22 deletions

diff --git a/util/testrunner/qt-testrunner.py b/util/testrunner/qt-testrunner.py
index 0020bd5fdf1..d0d54f054ca 100755
--- a/util/testrunner/qt-testrunner.py
+++ b/util/testrunner/qt-testrunner.py
@@ -64,9 +64,9 @@
 #   0: PASS. Either no test failed, or failed initially but passed
 #      in the re-runs (FLAKY PASS).
 #   1: Some unexpected error of this script.
-#   2: FAIL! for at least one test, even after the re-runs.
+#   2: FAIL! for at least one test, even after the individual re-runs.
 #   3: CRASH! for the test executable even after re-running it once.
-
+#        Or when we can't re-run individual functions for any reason.
 
 
 import sys
@@ -99,6 +99,10 @@ VERBOSE_ENV = {
     "QT_LOGGING_RULES": "*=true",
     "QT_MESSAGE_PATTERN": "[%{time process} %{if-debug}D%{endif}%{if-warning}W%{endif}%{if-critical}C%{endif}%{if-fatal}F%{endif}] %{category} %{file}:%{line} %{function}()  -  %{message}",
 }
+# The following special function names can not re-run individually.
+NO_RERUN_FUNCTIONS = {
+    "initTestCase", "init", "cleanup", "cleanupTestCase"
+}
 
 
 def parse_args():
@@ -300,39 +304,49 @@ def main():
     args = parse_args()
     n_full_runs = 1 if args.parse_xml_testlog else 2
 
-    for i in range(n_full_runs):
+    for i in range(n_full_runs + 1):
+
+        if 0 < i < n_full_runs:
+            L.info("Will re-run the full test executable")
+        elif i == n_full_runs:    # Failed on the final run
+            L.error("Full test run failed repeatedly, aborting!")
+            sys.exit(3)
+
         try:
-            if i != 0:
-                L.info("Re-running the full test!")
-            if args.parse_xml_testlog:
-                retcode = 1    # pretend the test returned error
-                results_file = args.parse_xml_testlog
-            else:
+            failed_functions = []
+            if args.parse_xml_testlog:      # do not run test, just parse file
+                failed_functions = parse_log(args.parse_xml_testlog)
+                # Pretend the test returned correct exit code
+                retcode = len(failed_functions)
+            else:                                # normal invocation, run test
                 (retcode, results_file) = \
                     run_full_test(args.test_basename, args.testargs, args.log_dir,
                                   args.no_extra_args, args.dry_run, args.timeout,
                                   args.specific_extra_args)
-                if retcode == 0:
-                    sys.exit(0)    # PASS
+                if retcode != 0 and results_file:
+                    failed_functions = parse_log(results_file)
+
+            if retcode == 0:
+                sys.exit(0)    # PASS
 
-            failed_functions = parse_log(results_file)
+            if len(failed_functions) == 0:
+                L.info("No failures listed in the XML test log!"
+                       " Did the test CRASH right after all its testcases PASSed?")
+                continue
 
-            if not args.parse_xml_testlog:
-                assert len(failed_functions) > 0, \
-                    "The XML test log should contain at least one failure!" \
-                    " Did the test CRASH right after all its testcases PASSed?"
+            cant_rerun = [ f.func for f in failed_functions if f.func in NO_RERUN_FUNCTIONS ]
+            if cant_rerun:
+                L.info(f"Failure detected in the special test function '{cant_rerun[0]}'"
+                       " which can not be re-run individually")
+                continue
 
-            break    # go to re-running individual failed testcases
+            assert len(failed_functions) > 0  and  retcode != 0
+            break    # all is fine, goto re-running individual failed testcases
 
         except Exception as e:
             L.exception("The test executable CRASHed uncontrollably!"
                         " Details about where we caught the problem:",
                         exc_info=e)
-            if i < n_full_runs - 1:
-                L.info("Will re-run the full test executable")
-            else:    # Failed on the final run
-                L.error("Full test run failed repeatedly, aborting!")
-                sys.exit(3)
 
     if args.max_repeats == 0:
         sys.exit(2)          # Some tests failed but no re-runs were asked