qt-testrunner: check validity of the XML file on test function re-runs

Tests on certain platforms run with specific wrappers, that might have
trouble reporting back the proper exit code.

For example, on Android each test is transfered to the device/emulator
and is executed not as a separate process, but as an activity. The
equivalent of "exit code" is caught and returned by a special wrapper
script for the platform.

It happens sometimes that these wrapper scripts fail to report back
correctly, and report zero (0) despite failed tests. For that reason we
now parse the test XML log on individual test re-runs too, and reporting
inconsistencies as CRASH, like we do with the main test execution.

Task-number: QTQAINFRA-7349
Task-number: QTQAINFRA-7378
Pick-to: 6.11 6.10 6.8
Change-Id: I27525f22331d44141be8825786a6f71e89543e92
Reviewed-by: Edward Welbourne <[email protected]>

2 files changed, 31 insertions, 11 deletions

diff --git a/tests/auto/util/testrunner/qt_mock_test.py b/tests/auto/util/testrunner/qt_mock_test.py
index ff4640ac3b8..af8fdf24509 100755
--- a/tests/auto/util/testrunner/qt_mock_test.py
+++ b/tests/auto/util/testrunner/qt_mock_test.py
@@ -106,6 +106,15 @@ def log_test(testcase, result,
              testsuite=MY_NAME.rpartition(".")[0]):
     print("%-7s: %s::%s()" % (result, testsuite, testcase))
 
+def log_xml(fail_list):
+    if XML_OUTPUT_FILE and XML_TEMPLATE is not None:
+        if XML_TEMPLATE == "":
+            # If the template is an empty file, then write an empty output file
+            with open(XML_OUTPUT_FILE, "w"):
+                pass
+        else:
+            write_xml_log(XML_OUTPUT_FILE, failure=fail_list)
+
 # Return the exit code
 def run_test(testname):
     if   testname == "initTestCase":
@@ -159,13 +168,7 @@ def no_args_run():
             fail_list.append(test)
         total_result = total_result and (test_exit_code == 0)
 
-    if XML_OUTPUT_FILE:
-        if XML_TEMPLATE:
-            write_xml_log(XML_OUTPUT_FILE, failure=fail_list)
-        # If the template is an empty file, then write an empty output file
-        elif XML_TEMPLATE == "":
-            with open(XML_OUTPUT_FILE, "w"):
-                pass
+    log_xml(fail_list)
 
     if CRASH_CLEANLY:
         # Crash despite all going well and writing all output files cleanly.
@@ -198,8 +201,14 @@ def main():
     if len(args) == 0:
         no_args_run()
         assert False, "Unreachable!"
-    else:
-        sys.exit(run_test(args[0]))
+    else:                                           # run single test function
+        exit_code = run_test(args[0])
+        # Write "fail" in the XML log only if the specific run has failed.
+        if exit_code != 0:
+            log_xml([args[0]])
+        else:
+            log_xml([])
+        sys.exit(exit_code)
 
 
 # TODO write XPASS test that does exit(1)
diff --git a/util/testrunner/qt-testrunner.py b/util/testrunner/qt-testrunner.py
index 37652f736fa..7928cd5f487 100755
--- a/util/testrunner/qt-testrunner.py
+++ b/util/testrunner/qt-testrunner.py
@@ -385,8 +385,19 @@ def rerun_failed_testcase(test_basename, testargs: List[str], output_dir: str,
             proc = run_test(testargs + output_args + VERBOSE_ARGS + [failed_arg],
                             timeout=timeout,
                             env={**os.environ, **VERBOSE_ENV})
+        # There are platforms that run tests wrapped with some test-runner
+        # script, that can possibly fail to extract a process exit code.
+        # Because of these cases, we *also* parse the XML file and signify
+        # CRASH in case of QFATAL/empty/corrupt result.
+        what_failed = parse_log(f"{pathname_stem}.xml")
+        if what_failed.qfatal_message:
+            raise ReRunCrash(f"CRASH! returncode:{proc.returncode} "
+                             f"QFATAL:'{what_failed.qfatal_message}'")
         if proc.returncode < 0 or proc.returncode >= 128:
             raise ReRunCrash(f"CRASH! returncode:{proc.returncode}")
+        if proc.returncode == 0 and len(what_failed.failed_tests) > 0:
+            raise ReRunCrash("CRASH! returncode:0 but failures were found: "
+                             + what_failed.failed_tests)
         if proc.returncode == 0:
             n_passes += 1
         if n_passes == passes_needed:
@@ -471,9 +482,9 @@ def main():
             ret = rerun_failed_testcase(args.test_basename, args.testargs, args.log_dir,
                                         test_result, args.max_repeats, args.passes_needed,
                                         dryrun=args.dry_run, timeout=args.timeout)
-        except ReRunCrash as e:
+        except Exception as e:
             L.error("exception:%s", e)
-            L.error("The testcase re-run crashed, giving up")
+            L.error("The testcase re-run probably crashed, giving up")
             sys.exit(3)                                    # Test re-run CRASH
 
         if not ret: