QPlainTestLogger: don't print "RESULT" before each additional result

We do that by passing the full list of results to the logger, to a
virtual that is present in the base class to call the existing
function. For all but the plain logger, we'll just print multiple
results. The plain logger now prints:

RESULT : tst_MyClass::QString_toInt()
     383 nsecs per iteration (total: 3,837,324, iterations: 10000)
     1,069 CPU cycles per iteration (total: 10,692,457, iterations: 10000)
     3,123 instructions per iteration (total: 31,230,101, iterations: 10000)
     536 branch instructions per iteration (total: 5,360,022, iterations: 10000)

Change-Id: I3c79b7e08fa346988dfefffd17203cb5802693dd
Reviewed-by: Qt CI Bot <[email protected]>
Reviewed-by: Fabian Kosmale <[email protected]>

7 files changed, 52 insertions, 28 deletions

diff --git a/src/testlib/qabstracttestlogger.cpp b/src/testlib/qabstracttestlogger.cpp
index 08fdade90bd..596799f5c44 100644
--- a/src/testlib/qabstracttestlogger.cpp
+++ b/src/testlib/qabstracttestlogger.cpp
@@ -3,6 +3,7 @@
 
 #include <QtTest/private/qabstracttestlogger_p.h>
 #include <QtTest/qtestassert.h>
+#include <qbenchmark_p.h>
 #include <qtestresult_p.h>
 
 #include <QtCore/qbytearray.h>
@@ -222,6 +223,12 @@ void QAbstractTestLogger::stopLogging()
 {
 }
 
+void QAbstractTestLogger::addBenchmarkResults(const QList<QBenchmarkResult> &result)
+{
+    for (const auto &m : result)
+        addBenchmarkResult(m);
+}
+
 /*!
     \fn void QAbstractTestLogger::enterTestFunction(const char *function)
 
diff --git a/src/testlib/qabstracttestlogger_p.h b/src/testlib/qabstracttestlogger_p.h
index d913bb649f5..188967981cc 100644
--- a/src/testlib/qabstracttestlogger_p.h
+++ b/src/testlib/qabstracttestlogger_p.h
@@ -68,6 +68,7 @@ public:
     virtual void addIncident(IncidentTypes type, const char *description,
                              const char *file = nullptr, int line = 0) = 0;
     virtual void addBenchmarkResult(const QBenchmarkResult &result) = 0;
+    virtual void addBenchmarkResults(const QList<QBenchmarkResult> &result);
 
     virtual void addMessage(QtMsgType, const QMessageLogContext &,
                             const QString &);
diff --git a/src/testlib/qplaintestlogger.cpp b/src/testlib/qplaintestlogger.cpp
index 0c7b4fdb460..a0d2655a6bb 100644
--- a/src/testlib/qplaintestlogger.cpp
+++ b/src/testlib/qplaintestlogger.cpp
@@ -47,6 +47,11 @@ template <int N> struct FixedBufString
     std::array<char, N + 2> buf;    // for the newline and terminating null
     FixedBufString()
     {
+        clear();
+    }
+    void clear()
+    {
+        used = 0;
         buf[0] = '\0';
     }
 
@@ -265,29 +270,38 @@ void QPlainTestLogger::printMessage(MessageSource source, const char *type, cons
     outputMessage(messagePrefix.data());
 }
 
-void QPlainTestLogger::printBenchmarkResult(const QBenchmarkResult &result)
+void QPlainTestLogger::printBenchmarkResultsHeader(const QBenchmarkResult &result)
 {
     FixedBufString<1022> buf;
-
     buf.appendf("%s: %s::%s", QTest::benchmarkResult2String(),
                 QTestResult::currentTestObjectName(), result.context.slotName.toLatin1().data());
 
     if (QByteArray tag = result.context.tag.toLocal8Bit(); !tag.isEmpty())
-        buf.appendf(":\"%s\"", tag.data());
+        buf.appendf(":\"%s\":\n", tag.data());
+    else
+        buf.append(":\n");
+    outputMessage(buf);
+}
 
-    const char * unitText = QTest::benchmarkMetricUnit(result.measurement.metric);
-    int significantDigits = QTest::countSignificantDigits(result.measurement.value);
-    qreal valuePerIteration = qreal(result.measurement.value) / qreal(result.iterations);
-    buf.appendf(":\n     %s %s%s",
-                QTest::formatResult(valuePerIteration, significantDigits).constData(),
-                unitText, result.setByMacro ? " per iteration" : "");
+void QPlainTestLogger::printBenchmarkResults(const QList<QBenchmarkResult> &results)
+{
+    FixedBufString<1022> buf;
+    for (const QBenchmarkResult &result : results) {
+        buf.clear();
 
-    Q_ASSERT(result.iterations > 0);
-    buf.appendf(" (total: %s, iterations: %d)",
-                QTest::formatResult(result.measurement.value, significantDigits).constData(),
-                result.iterations);
+        const char * unitText = QTest::benchmarkMetricUnit(result.measurement.metric);
+        int significantDigits = QTest::countSignificantDigits(result.measurement.value);
+        qreal valuePerIteration = qreal(result.measurement.value) / qreal(result.iterations);
+        buf.appendf("     %s %s%s", QTest::formatResult(valuePerIteration, significantDigits).constData(),
+                    unitText, result.setByMacro ? " per iteration" : "");
 
-    outputMessage(buf);
+        Q_ASSERT(result.iterations > 0);
+        buf.appendf(" (total: %s, iterations: %d)\n",
+                    QTest::formatResult(result.measurement.value, significantDigits).constData(),
+                    result.iterations);
+
+        outputMessage(buf);
+    }
 }
 
 QPlainTestLogger::QPlainTestLogger(const char *filename)
@@ -357,13 +371,14 @@ void QPlainTestLogger::addIncident(IncidentTypes type, const char *description,
     printMessage(MessageSource::Incident, QTest::incidentType2String(type), description, file, line);
 }
 
-void QPlainTestLogger::addBenchmarkResult(const QBenchmarkResult &result)
+void QPlainTestLogger::addBenchmarkResults(const QList<QBenchmarkResult> &results)
 {
     // suppress benchmark results in silent mode
-    if (QTestLog::verboseLevel() < 0)
+    if (QTestLog::verboseLevel() < 0 || results.isEmpty())
         return;
 
-    printBenchmarkResult(result);
+    printBenchmarkResultsHeader(results.first());
+    printBenchmarkResults(results);
 }
 
 void QPlainTestLogger::addMessage(QtMsgType type, const QMessageLogContext &context, const QString &message)
diff --git a/src/testlib/qplaintestlogger_p.h b/src/testlib/qplaintestlogger_p.h
index ddb406f2617..13228a7631f 100644
--- a/src/testlib/qplaintestlogger_p.h
+++ b/src/testlib/qplaintestlogger_p.h
@@ -33,7 +33,9 @@ public:
 
     void addIncident(IncidentTypes type, const char *description,
                      const char *file = nullptr, int line = 0) override;
-    void addBenchmarkResult(const QBenchmarkResult &result) override;
+    void addBenchmarkResult(const QBenchmarkResult &) final override
+    { Q_UNREACHABLE(); }
+    void addBenchmarkResults(const QList<QBenchmarkResult> &results) override;
 
     void addMessage(QtMsgType, const QMessageLogContext &,
                     const QString &) override;
@@ -49,7 +51,8 @@ private:
     void printMessage(MessageSource source, const char *type, const char *msg,
                       const char *file = nullptr, int line = 0);
     void outputMessage(const char *str);
-    void printBenchmarkResult(const QBenchmarkResult &result);
+    void printBenchmarkResultsHeader(const QBenchmarkResult &result);
+    void printBenchmarkResults(const QList<QBenchmarkResult> &result);
 };
 
 QT_END_NAMESPACE
diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp
index 63691b0c0bd..62512dc9d38 100644
--- a/src/testlib/qtestcase.cpp
+++ b/src/testlib/qtestcase.cpp
@@ -1202,12 +1202,8 @@ void TestMethods::invokeTestOnData(int index) const
         bool testPassed = !QTestResult::skipCurrentTest() && !QTestResult::currentTestFailed();
         QTestResult::finishedCurrentTestDataCleanup();
         // Only report benchmark figures if the test passed
-        if (testPassed && QBenchmarkTestMethodData::current->resultsAccepted()) {
-            const QList<QBenchmarkResult> median = qMedian(resultsList);
-            for (auto m : median) {
-                QTestLog::addBenchmarkResult(m);
-            }
-        }
+        if (testPassed && QBenchmarkTestMethodData::current->resultsAccepted())
+            QTestLog::addBenchmarkResults(qMedian(resultsList));
     }
 }
 
diff --git a/src/testlib/qtestlog.cpp b/src/testlib/qtestlog.cpp
index 3ec695679c6..dca4bc1c109 100644
--- a/src/testlib/qtestlog.cpp
+++ b/src/testlib/qtestlog.cpp
@@ -471,10 +471,10 @@ void QTestLog::addSkip(const char *msg, const char *file, int line)
         logger->addIncident(QAbstractTestLogger::Skip, msg, file, line);
 }
 
-void QTestLog::addBenchmarkResult(const QBenchmarkResult &result)
+void QTestLog::addBenchmarkResults(const QList<QBenchmarkResult> &results)
 {
     FOREACH_TEST_LOGGER
-        logger->addBenchmarkResult(result);
+        logger->addBenchmarkResults(results);
 }
 
 void QTestLog::startLogging()
diff --git a/src/testlib/qtestlog_p.h b/src/testlib/qtestlog_p.h
index 8852609d7aa..9717858afb6 100644
--- a/src/testlib/qtestlog_p.h
+++ b/src/testlib/qtestlog_p.h
@@ -63,7 +63,9 @@ public:
     static void addBXPass(const char *msg, const char *file, int line);
     static void addBXFail(const char *msg, const char *file, int line);
     static void addSkip(const char *msg, const char *file, int line);
-    static void addBenchmarkResult(const QBenchmarkResult &result);
+    static void addBenchmarkResult(const QList<QBenchmarkResult> &result)
+    { return addBenchmarkResults({ result }); }
+    static void addBenchmarkResults(const QList<QBenchmarkResult> &result);
 
     static void ignoreMessage(QtMsgType type, const char *msg);
 #ifndef QT_NO_REGULAREXPRESSION