Skip early return from test loops during cleanup()

The QTRY_* macros and QTestEventLoop exit early if the test has
resolved; however, in the cleanup phase of a test, even if the test
has failed, these loops should continue as normal.

[ChangeLog][QtTest] During the cleanup() phase of a test, the QTRY_*
macros and QTestEventLoop now ignore the test resolution, in contrast
to when they are used from the test itself, which (since 6.3.0) exits
the loops early if the test has failed.

Pick-to: 6.4 6.3
Fixes: QTBUG-104441
Change-Id: I2673161967cbbc57815155af698a9338ab98a686
Reviewed-by: Jason McDonald <[email protected]>

9 files changed, 39 insertions, 119 deletions

diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp
index 99be5afaefa..2c9f2d4049e 100644
--- a/src/testlib/qtestcase.cpp
+++ b/src/testlib/qtestcase.cpp
@@ -468,6 +468,7 @@ class WatchDog;
 
 static QObject *currentTestObject = nullptr;
 static QString mainSourcePath;
+static bool inTestFunction = false;
 
 #if defined(Q_OS_MACOS)
 static IOPMAssertionID macPowerSavingDisabled = 0;
@@ -1120,6 +1121,7 @@ void TestMethods::invokeTestOnData(int index) const
         /* Benchmarking: for each accumulation iteration*/
         bool invokeOk;
         do {
+            QTest::inTestFunction = true;
             if (m_initMethod.isValid())
                 m_initMethod.invoke(QTest::currentTestObject, Qt::DirectConnection);
 
@@ -1141,6 +1143,7 @@ void TestMethods::invokeTestOnData(int index) const
                 invokeOk = false;
             }
 
+            QTest::inTestFunction = false;
             QTestResult::finishedCurrentTestData();
 
             if (!initQuit) {
@@ -3012,6 +3015,17 @@ bool QTest::currentTestFailed()
     return QTestResult::currentTestFailed();
 }
 
+/*
+    Returns \c true during the run of the test-function and its set-up.
+
+    Used by the \c{QTRY_*} macros and \l QTestEventLoop to check whether to
+    return when QTest::currentTestFailed() is true.
+*/
+bool QTest::runningTest()
+{
+    return QTest::inTestFunction;
+}
+
 /*! \internal
 */
 QObject *QTest::testObject()
diff --git a/src/testlib/qtestcase.h b/src/testlib/qtestcase.h
index 3d268135a42..6affbf15076 100644
--- a/src/testlib/qtestcase.h
+++ b/src/testlib/qtestcase.h
@@ -148,9 +148,10 @@ inline void useVerifyThrowsException() {}
 /* Ideally we would adapt qWaitFor(), or a variant on it, to implement roughly
  * what the following provides as QTRY_LOOP_IMPL(); however, for now, the
  * reporting of how much to increase the timeout to (if within a factor of two)
- * on failure and the check for QTest::currentTestFailed() go beyond
- * qWaitFor(). (We no longer care about the bug in MSVC < 2017 that precluded
- * using qWaitFor() in the implementation here, see QTBUG-59096.)
+ * on failure and the check for (QTest::runningTest() &&
+ * QTest::currentTestFailed()) go beyond qWaitFor(). (We no longer care about
+ * the bug in MSVC < 2017 that precluded using qWaitFor() in the implementation
+ * here, see QTBUG-59096.)
  */
 
 // NB: not do {...} while (0) wrapped, as qt_test_i is accessed after it
@@ -159,14 +160,14 @@ inline void useVerifyThrowsException() {}
         QTest::qWait(0); \
     } \
     int qt_test_i = 0; \
-    for (; qt_test_i < timeoutValue && !QTest::currentTestFailed() \
+    for (; qt_test_i < timeoutValue && !(QTest::runningTest() && QTest::currentTestFailed()) \
              && !(expr); qt_test_i += step) { \
         QTest::qWait(step); \
     }
 // Ends in a for-block, so doesn't want a following semicolon.
 
 #define QTRY_TIMEOUT_DEBUG_IMPL(expr, timeoutValue, step) \
-    if (!QTest::currentTestFailed() && !(expr)) { \
+    if (!(QTest::runningTest() && QTest::currentTestFailed()) && !(expr)) { \
         QTRY_LOOP_IMPL(expr, 2 * (timeoutValue), step) \
         if ((expr)) { \
             QFAIL(qPrintable(QTest::Internal::formatTryTimeoutDebugMessage(\
@@ -419,6 +420,7 @@ namespace QTest
     Q_TESTLIB_EXPORT const char *currentTestFunction();
     Q_TESTLIB_EXPORT const char *currentDataTag();
     Q_TESTLIB_EXPORT bool currentTestFailed();
+    Q_TESTLIB_EXPORT bool runningTest(); // Internal, for use by macros and QTestEventLoop.
 
     Q_TESTLIB_EXPORT Qt::Key asciiToKey(char ascii);
     Q_TESTLIB_EXPORT char keyToAscii(Qt::Key key);
diff --git a/src/testlib/qtesteventloop.h b/src/testlib/qtesteventloop.h
index a8126bc3330..2ac8bdecdfb 100644
--- a/src/testlib/qtesteventloop.h
+++ b/src/testlib/qtesteventloop.h
@@ -60,7 +60,7 @@ inline void QTestEventLoop::enterLoopMSecs(int ms)
     Q_ASSERT(!loop);
     _timeout = false;
 
-    if (QTest::currentTestFailed())
+    if (QTest::runningTest() && QTest::currentTestFailed())
         return;
 
     QEventLoop l;
diff --git a/tests/auto/testlib/selftests/expected_eventloop.junitxml b/tests/auto/testlib/selftests/expected_eventloop.junitxml
index 15a21427c83..566bc1605ed 100644
--- a/tests/auto/testlib/selftests/expected_eventloop.junitxml
+++ b/tests/auto/testlib/selftests/expected_eventloop.junitxml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite name="tst_EventLoop" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="5" failures="3" errors="0" skipped="0" time="@TEST_DURATION@">
+<testsuite name="tst_EventLoop" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="5" failures="1" errors="0" skipped="1" time="@TEST_DURATION@">
   <properties>
     <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
     <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
@@ -10,10 +10,8 @@
     <failure type="fail" message="Failing test should still clean up"/>
   </testcase>
   <testcase name="skip" classname="tst_EventLoop" time="@TEST_DURATION@">
-    <failure type="fail" message="&apos;!std::exchange(m_inTestFunction, true)&apos; returned FALSE. (Earlier test failed to clean up)"/>
-  </testcase>
-  <testcase name="pass" classname="tst_EventLoop" time="@TEST_DURATION@">
-    <failure type="fail" message="&apos;!std::exchange(m_inTestFunction, true)&apos; returned FALSE. (Earlier test failed to clean up)"/>
+    <skipped message="Skipping test should still clean up"/>
   </testcase>
+  <testcase name="pass" classname="tst_EventLoop" time="@TEST_DURATION@"/>
   <testcase name="cleanupTestCase" classname="tst_EventLoop" time="@TEST_DURATION@"/>
 </testsuite>
diff --git a/tests/auto/testlib/selftests/expected_eventloop.lightxml b/tests/auto/testlib/selftests/expected_eventloop.lightxml
index 0d67d73f752..40880fde013 100644
--- a/tests/auto/testlib/selftests/expected_eventloop.lightxml
+++ b/tests/auto/testlib/selftests/expected_eventloop.lightxml
@@ -11,27 +11,16 @@
     <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
       <Description><![CDATA[Failing test should still clean up]]></Description>
     </Incident>
-    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
-      <Description><![CDATA['loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())]]></Description>
-    </Incident>
     <Duration msecs="0"/>
   </TestFunction>
   <TestFunction name="skip">
-    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
-      <Description><![CDATA['!std::exchange(m_inTestFunction, true)' returned FALSE. (Earlier test failed to clean up)]]></Description>
-    </Incident>
-    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
-      <Description><![CDATA['loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())]]></Description>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
+      <Description><![CDATA[Skipping test should still clean up]]></Description>
     </Incident>
     <Duration msecs="0"/>
   </TestFunction>
   <TestFunction name="pass">
-    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
-      <Description><![CDATA['!std::exchange(m_inTestFunction, true)' returned FALSE. (Earlier test failed to clean up)]]></Description>
-    </Incident>
-    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
-      <Description><![CDATA['loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())]]></Description>
-    </Incident>
+    <Incident type="pass" file="" line="0" />
     <Duration msecs="0"/>
   </TestFunction>
   <TestFunction name="cleanupTestCase">
diff --git a/tests/auto/testlib/selftests/expected_eventloop.tap b/tests/auto/testlib/selftests/expected_eventloop.tap
index 84036e6dec5..496a6e636a9 100644
--- a/tests/auto/testlib/selftests/expected_eventloop.tap
+++ b/tests/auto/testlib/selftests/expected_eventloop.tap
@@ -8,68 +8,10 @@ not ok 2 - fail()
   file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp
   line: 0
   ...
-not ok 2 - fail()
-  ---
-  type: QVERIFY
-  message: QTestEventLoop exited prematurely in cleanup()
-  wanted: true (loop.timeout())
-  found: false (loop.timeout())
-  expected: true (loop.timeout())
-  actual: false (loop.timeout())
-  at: tst_EventLoop::fail() (qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp:0)
-  file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp
-  line: 0
-  ...
-not ok 3 - skip()
-  ---
-  type: QVERIFY
-  message: Earlier test failed to clean up
-  wanted: true (!std::exchange(m_inTestFunction, true))
-  found: false (!std::exchange(m_inTestFunction, true))
-  expected: true (!std::exchange(m_inTestFunction, true))
-  actual: false (!std::exchange(m_inTestFunction, true))
-  at: tst_EventLoop::skip() (qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp:0)
-  file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp
-  line: 0
-  ...
-not ok 3 - skip()
-  ---
-  type: QVERIFY
-  message: QTestEventLoop exited prematurely in cleanup()
-  wanted: true (loop.timeout())
-  found: false (loop.timeout())
-  expected: true (loop.timeout())
-  actual: false (loop.timeout())
-  at: tst_EventLoop::skip() (qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp:0)
-  file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp
-  line: 0
-  ...
-not ok 4 - pass()
-  ---
-  type: QVERIFY
-  message: Earlier test failed to clean up
-  wanted: true (!std::exchange(m_inTestFunction, true))
-  found: false (!std::exchange(m_inTestFunction, true))
-  expected: true (!std::exchange(m_inTestFunction, true))
-  actual: false (!std::exchange(m_inTestFunction, true))
-  at: tst_EventLoop::pass() (qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp:0)
-  file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp
-  line: 0
-  ...
-not ok 4 - pass()
-  ---
-  type: QVERIFY
-  message: QTestEventLoop exited prematurely in cleanup()
-  wanted: true (loop.timeout())
-  found: false (loop.timeout())
-  expected: true (loop.timeout())
-  actual: false (loop.timeout())
-  at: tst_EventLoop::pass() (qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp:0)
-  file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp
-  line: 0
-  ...
+ok 3 - skip() # SKIP Skipping test should still clean up
+ok 4 - pass()
 ok 5 - cleanupTestCase()
 1..5
 # tests 5
-# pass 2
-# fail 3
+# pass 3
+# fail 1
diff --git a/tests/auto/testlib/selftests/expected_eventloop.teamcity b/tests/auto/testlib/selftests/expected_eventloop.teamcity
index 4c832be52f7..a293a201357 100644
--- a/tests/auto/testlib/selftests/expected_eventloop.teamcity
+++ b/tests/auto/testlib/selftests/expected_eventloop.teamcity
@@ -4,17 +4,10 @@
 ##teamcity[testStarted name='fail()' flowId='tst_EventLoop']
 ##teamcity[testFailed name='fail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='Failing test should still clean up' flowId='tst_EventLoop']
 ##teamcity[testFinished name='fail()' flowId='tst_EventLoop']
-##teamcity[testFailed name='fail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='|'loop.timeout()|' returned FALSE. (QTestEventLoop exited prematurely in cleanup())' flowId='tst_EventLoop']
-##teamcity[testFinished name='fail()' flowId='tst_EventLoop']
 ##teamcity[testStarted name='skip()' flowId='tst_EventLoop']
-##teamcity[testFailed name='skip()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='|'!std::exchange(m_inTestFunction, true)|' returned FALSE. (Earlier test failed to clean up)' flowId='tst_EventLoop']
-##teamcity[testFinished name='skip()' flowId='tst_EventLoop']
-##teamcity[testFailed name='skip()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='|'loop.timeout()|' returned FALSE. (QTestEventLoop exited prematurely in cleanup())' flowId='tst_EventLoop']
+##teamcity[testIgnored name='skip()' message='Skipping test should still clean up |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' flowId='tst_EventLoop']
 ##teamcity[testFinished name='skip()' flowId='tst_EventLoop']
 ##teamcity[testStarted name='pass()' flowId='tst_EventLoop']
-##teamcity[testFailed name='pass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='|'!std::exchange(m_inTestFunction, true)|' returned FALSE. (Earlier test failed to clean up)' flowId='tst_EventLoop']
-##teamcity[testFinished name='pass()' flowId='tst_EventLoop']
-##teamcity[testFailed name='pass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='|'loop.timeout()|' returned FALSE. (QTestEventLoop exited prematurely in cleanup())' flowId='tst_EventLoop']
 ##teamcity[testFinished name='pass()' flowId='tst_EventLoop']
 ##teamcity[testStarted name='cleanupTestCase()' flowId='tst_EventLoop']
 ##teamcity[testFinished name='cleanupTestCase()' flowId='tst_EventLoop']
diff --git a/tests/auto/testlib/selftests/expected_eventloop.txt b/tests/auto/testlib/selftests/expected_eventloop.txt
index b614c876c6d..548ef393f6d 100644
--- a/tests/auto/testlib/selftests/expected_eventloop.txt
+++ b/tests/auto/testlib/selftests/expected_eventloop.txt
@@ -3,16 +3,9 @@ Config: Using QtTest library
 PASS   : tst_EventLoop::initTestCase()
 FAIL!  : tst_EventLoop::fail() Failing test should still clean up
    Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)]
-FAIL!  : tst_EventLoop::fail() 'loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())
-   Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)]
-FAIL!  : tst_EventLoop::skip() '!std::exchange(m_inTestFunction, true)' returned FALSE. (Earlier test failed to clean up)
-   Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)]
-FAIL!  : tst_EventLoop::skip() 'loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())
-   Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)]
-FAIL!  : tst_EventLoop::pass() '!std::exchange(m_inTestFunction, true)' returned FALSE. (Earlier test failed to clean up)
-   Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)]
-FAIL!  : tst_EventLoop::pass() 'loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())
+SKIP   : tst_EventLoop::skip() Skipping test should still clean up
    Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)]
+PASS   : tst_EventLoop::pass()
 PASS   : tst_EventLoop::cleanupTestCase()
-Totals: 2 passed, 3 failed, 0 skipped, 0 blacklisted, 0ms
+Totals: 3 passed, 1 failed, 1 skipped, 0 blacklisted, 0ms
 ********* Finished testing of tst_EventLoop *********
diff --git a/tests/auto/testlib/selftests/expected_eventloop.xml b/tests/auto/testlib/selftests/expected_eventloop.xml
index 2613a734494..f9d94756668 100644
--- a/tests/auto/testlib/selftests/expected_eventloop.xml
+++ b/tests/auto/testlib/selftests/expected_eventloop.xml
@@ -13,27 +13,16 @@
     <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
       <Description><![CDATA[Failing test should still clean up]]></Description>
     </Incident>
-    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
-      <Description><![CDATA['loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())]]></Description>
-    </Incident>
     <Duration msecs="0"/>
   </TestFunction>
   <TestFunction name="skip">
-    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
-      <Description><![CDATA['!std::exchange(m_inTestFunction, true)' returned FALSE. (Earlier test failed to clean up)]]></Description>
-    </Incident>
-    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
-      <Description><![CDATA['loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())]]></Description>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
+      <Description><![CDATA[Skipping test should still clean up]]></Description>
     </Incident>
     <Duration msecs="0"/>
   </TestFunction>
   <TestFunction name="pass">
-    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
-      <Description><![CDATA['!std::exchange(m_inTestFunction, true)' returned FALSE. (Earlier test failed to clean up)]]></Description>
-    </Incident>
-    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
-      <Description><![CDATA['loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())]]></Description>
-    </Incident>
+    <Incident type="pass" file="" line="0" />
     <Duration msecs="0"/>
   </TestFunction>
   <TestFunction name="cleanupTestCase">