From aecdcfb11de8d82bf7cb5bb5d0b327bfa058da5b Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 31 Mar 2025 23:12:58 +0300 Subject: [PATCH 01/10] [conftest] Advanced processing of logging This patch does the following things: - it processes the calls of logging.error as test errors - it prints the number of errors/warnings for each test --- tests/conftest.py | 212 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 209 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9e8ea368..8183549c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,6 +8,7 @@ import pathlib import math import datetime +import typing import _pytest.outcomes import _pytest.unittest @@ -212,6 +213,12 @@ def helper__build_test_id(item: pytest.Function) -> str: return testID + +# ///////////////////////////////////////////////////////////////////////////// + +g_error_msg_count_key = pytest.StashKey[int]() +g_warning_msg_count_key = pytest.StashKey[int]() + # ///////////////////////////////////////////////////////////////////////////// @@ -285,6 +292,16 @@ def helper__makereport__call( assert type(call) == pytest.CallInfo # noqa: E721 assert type(outcome) == pluggy.Result # noqa: E721 + # -------- + item_error_msg_count = item.stash.get(g_error_msg_count_key, 0) + assert type(item_error_msg_count) == int # noqa: E721 + assert item_error_msg_count >= 0 + + item_warning_msg_count = item.stash.get(g_warning_msg_count_key, 0) + assert type(item_warning_msg_count) == int # noqa: E721 + assert item_warning_msg_count >= 0 + + # -------- rep = outcome.get_result() assert rep is not None assert type(rep) == pytest.TestReport # noqa: E721 @@ -336,6 +353,7 @@ def helper__makereport__call( reasonMsgTempl = "XFAIL REASON: {0}" logging.error(call.excinfo.value) + item_error_msg_count += 1 assert type(reasonText) == str # noqa: E721 @@ -350,7 +368,13 @@ def helper__makereport__call( TEST_PROCESS_STATS.incrementFailedTestCount(testID) - logging.error(call.excinfo.value) + if type(call.excinfo.value) == SIGNAL_EXCEPTION: # noqa: E721 + assert item_error_msg_count > 0 + pass + else: + logging.error(call.excinfo.value) + item_error_msg_count += 1 + exitStatus = "FAILED" elif rep.outcome == "passed": assert call.excinfo is None @@ -380,9 +404,11 @@ def helper__makereport__call( # -------- logging.info("*") - logging.info("* DURATION : {0}".format(timedelta_to_human_text(testDurration))) + logging.info("* DURATION : {0}".format(timedelta_to_human_text(testDurration))) logging.info("*") - logging.info("* EXIT STATUS : {0}".format(exitStatus)) + logging.info("* EXIT STATUS : {0}".format(exitStatus)) + logging.info("* ERROR COUNT : {0}".format(item_error_msg_count)) + logging.info("* WARNING COUNT: {0}".format(item_warning_msg_count)) logging.info("*") logging.info("* STOP TEST {0}".format(testID)) logging.info("*") @@ -437,6 +463,186 @@ def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): # ///////////////////////////////////////////////////////////////////////////// +class LogErrorWrapper2: + _old_method: any + _counter: typing.Optional[int] + + # -------------------------------------------------------------------- + def __init__(self): + self._old_method = None + self._counter = None + + # -------------------------------------------------------------------- + def __enter__(self): + assert self._old_method is None + assert self._counter is None + + self._old_method = logging.error + self._counter = 0 + + logging.error = self + return self + + # -------------------------------------------------------------------- + def __exit__(self, exc_type, exc_val, exc_tb): + assert self._old_method is not None + assert self._counter is not None + + assert logging.error is self + + logging.error = self._old_method + + self._old_method = None + self._counter = None + return False + + # -------------------------------------------------------------------- + def __call__(self, *args, **kwargs): + assert self._old_method is not None + assert self._counter is not None + + assert type(self._counter) == int # noqa: E721 + assert self._counter >= 0 + + r = self._old_method(*args, **kwargs) + + self._counter += 1 + assert self._counter > 0 + + return r + + +# ///////////////////////////////////////////////////////////////////////////// + + +class LogWarningWrapper2: + _old_method: any + _counter: typing.Optional[int] + + # -------------------------------------------------------------------- + def __init__(self): + self._old_method = None + self._counter = None + + # -------------------------------------------------------------------- + def __enter__(self): + assert self._old_method is None + assert self._counter is None + + self._old_method = logging.warning + self._counter = 0 + + logging.warning = self + return self + + # -------------------------------------------------------------------- + def __exit__(self, exc_type, exc_val, exc_tb): + assert self._old_method is not None + assert self._counter is not None + + assert logging.warning is self + + logging.warning = self._old_method + + self._old_method = None + self._counter = None + return False + + # -------------------------------------------------------------------- + def __call__(self, *args, **kwargs): + assert self._old_method is not None + assert self._counter is not None + + assert type(self._counter) == int # noqa: E721 + assert self._counter >= 0 + + r = self._old_method(*args, **kwargs) + + self._counter += 1 + assert self._counter > 0 + + return r + + +# ///////////////////////////////////////////////////////////////////////////// + + +class SIGNAL_EXCEPTION(Exception): + def __init__(self): + pass + + +# ///////////////////////////////////////////////////////////////////////////// + + +@pytest.hookimpl(hookwrapper=True) +def pytest_pyfunc_call(pyfuncitem: pytest.Function): + assert pyfuncitem is not None + assert isinstance(pyfuncitem, pytest.Function) + + debug__log_error_method = logging.error + assert debug__log_error_method is not None + + debug__log_warning_method = logging.warning + assert debug__log_warning_method is not None + + pyfuncitem.stash[g_error_msg_count_key] = 0 + pyfuncitem.stash[g_warning_msg_count_key] = 0 + + try: + with LogErrorWrapper2() as logErrorWrapper, LogWarningWrapper2() as logWarningWrapper: + assert type(logErrorWrapper) == LogErrorWrapper2 # noqa: E721 + assert logErrorWrapper._old_method is not None + assert type(logErrorWrapper._counter) == int # noqa: E721 + assert logErrorWrapper._counter == 0 + assert logging.error is logErrorWrapper + + assert type(logWarningWrapper) == LogWarningWrapper2 # noqa: E721 + assert logWarningWrapper._old_method is not None + assert type(logWarningWrapper._counter) == int # noqa: E721 + assert logWarningWrapper._counter == 0 + assert logging.warning is logWarningWrapper + + r: pluggy.Result = yield + + assert r is not None + assert type(r) == pluggy.Result # noqa: E721 + + assert logErrorWrapper._old_method is not None + assert type(logErrorWrapper._counter) == int # noqa: E721 + assert logErrorWrapper._counter >= 0 + assert logging.error is logErrorWrapper + + assert logWarningWrapper._old_method is not None + assert type(logWarningWrapper._counter) == int # noqa: E721 + assert logWarningWrapper._counter >= 0 + assert logging.warning is logWarningWrapper + + assert g_error_msg_count_key in pyfuncitem.stash + assert g_warning_msg_count_key in pyfuncitem.stash + + assert pyfuncitem.stash[g_error_msg_count_key] == 0 + assert pyfuncitem.stash[g_warning_msg_count_key] == 0 + + pyfuncitem.stash[g_error_msg_count_key] = logErrorWrapper._counter + pyfuncitem.stash[g_warning_msg_count_key] = logWarningWrapper._counter + + if r.exception is not None: + pass + elif logErrorWrapper._counter == 0: + pass + else: + assert logErrorWrapper._counter > 0 + r.force_exception(SIGNAL_EXCEPTION()) + finally: + assert logging.error is debug__log_error_method + assert logging.warning is debug__log_warning_method + pass + + +# ///////////////////////////////////////////////////////////////////////////// + + def helper__calc_W(n: int) -> int: assert n > 0 From 88bb6ae1592f1209f64745d499f2643f10e591f0 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 1 Apr 2025 12:00:29 +0300 Subject: [PATCH 02/10] [conftest] XFail does not respect SIGNAL_EXCEPTION --- tests/conftest.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 8183549c..87ce204e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -352,8 +352,11 @@ def helper__makereport__call( reasonText = rep.wasxfail reasonMsgTempl = "XFAIL REASON: {0}" - logging.error(call.excinfo.value) - item_error_msg_count += 1 + if type(call.excinfo.value) == SIGNAL_EXCEPTION: # noqa: E721 + pass + else: + logging.error(call.excinfo.value) + item_error_msg_count += 1 assert type(reasonText) == str # noqa: E721 From 24b20c837426ea020be77dc8586f9624925310e0 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 1 Apr 2025 12:16:38 +0300 Subject: [PATCH 03/10] [summary] An error count of each test is printed --- tests/conftest.py | 92 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 72 insertions(+), 20 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 87ce204e..45404d97 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -109,8 +109,8 @@ class TEST_PROCESS_STATS: cUnexpectedTests: int = 0 cAchtungTests: int = 0 - FailedTests = list[str]() - XFailedTests = list[str]() + FailedTests = list[str, int]() + XFailedTests = list[str, int]() NotXFailedTests = list[str]() AchtungTests = list[str]() @@ -132,19 +132,23 @@ def incrementPassedTestCount() -> None: __class__.cPassedTests += 1 # -------------------------------------------------------------------- - def incrementFailedTestCount(testID: str) -> None: + def incrementFailedTestCount(testID: str, errCount: int) -> None: assert type(testID) == str # noqa: E721 + assert type(errCount) == int # noqa: E721 + assert errCount >= 0 assert type(__class__.FailedTests) == list # noqa: E721 - __class__.FailedTests.append(testID) # raise? + __class__.FailedTests.append((testID, errCount)) # raise? __class__.cFailedTests += 1 # -------------------------------------------------------------------- - def incrementXFailedTestCount(testID: str) -> None: + def incrementXFailedTestCount(testID: str, errCount: int) -> None: assert type(testID) == str # noqa: E721 + assert type(errCount) == int # noqa: E721 + assert errCount >= 0 assert type(__class__.XFailedTests) == list # noqa: E721 - __class__.XFailedTests.append(testID) # raise? + __class__.XFailedTests.append((testID, errCount)) # raise? __class__.cXFailedTests += 1 # -------------------------------------------------------------------- @@ -329,26 +333,25 @@ def helper__makereport__call( if type(call.excinfo.value) == _pytest.outcomes.Skipped: # noqa: E721 assert not hasattr(rep, "wasxfail") - TEST_PROCESS_STATS.incrementSkippedTestCount() - exitStatus = "SKIPPED" reasonText = str(call.excinfo.value) reasonMsgTempl = "SKIP REASON: {0}" - elif type(call.excinfo.value) == _pytest.outcomes.XFailed: # noqa: E721 - TEST_PROCESS_STATS.incrementXFailedTestCount(testID) + TEST_PROCESS_STATS.incrementSkippedTestCount() + elif type(call.excinfo.value) == _pytest.outcomes.XFailed: # noqa: E721 exitStatus = "XFAILED" reasonText = str(call.excinfo.value) reasonMsgTempl = "XFAIL REASON: {0}" + + TEST_PROCESS_STATS.incrementXFailedTestCount(testID, item_error_msg_count) + else: exitStatus = "XFAILED" assert hasattr(rep, "wasxfail") assert rep.wasxfail is not None assert type(rep.wasxfail) == str # noqa: E721 - TEST_PROCESS_STATS.incrementXFailedTestCount(testID) - reasonText = rep.wasxfail reasonMsgTempl = "XFAIL REASON: {0}" @@ -358,6 +361,8 @@ def helper__makereport__call( logging.error(call.excinfo.value) item_error_msg_count += 1 + TEST_PROCESS_STATS.incrementXFailedTestCount(testID, item_error_msg_count) + assert type(reasonText) == str # noqa: E721 if reasonText != "": @@ -369,8 +374,6 @@ def helper__makereport__call( assert call.excinfo is not None assert call.excinfo.value is not None - TEST_PROCESS_STATS.incrementFailedTestCount(testID) - if type(call.excinfo.value) == SIGNAL_EXCEPTION: # noqa: E721 assert item_error_msg_count > 0 pass @@ -378,6 +381,8 @@ def helper__makereport__call( logging.error(call.excinfo.value) item_error_msg_count += 1 + TEST_PROCESS_STATS.incrementFailedTestCount(testID, item_error_msg_count) + exitStatus = "FAILED" elif rep.outcome == "passed": assert call.excinfo is None @@ -676,11 +681,42 @@ def helper__print_test_list(tests: list[str]) -> None: nTest = 0 - while nTest < len(tests): - testID = tests[nTest] - assert type(testID) == str # noqa: E721 + for t in tests: + assert type(t) == str # noqa: E721 + assert t != "" nTest += 1 - logging.info(templateLine.format(nTest, testID)) + logging.info(templateLine.format(nTest, t)) + + +# ------------------------------------------------------------------------ +def helper__print_test_list2(tests: list[str, int]) -> None: + assert type(tests) == list # noqa: E721 + + assert helper__calc_W(9) == 1 + assert helper__calc_W(10) == 2 + assert helper__calc_W(11) == 2 + assert helper__calc_W(99) == 2 + assert helper__calc_W(100) == 3 + assert helper__calc_W(101) == 3 + assert helper__calc_W(999) == 3 + assert helper__calc_W(1000) == 4 + assert helper__calc_W(1001) == 4 + + W = helper__calc_W(len(tests)) + + templateLine = "{0:0" + str(W) + "d}. {1} ({2})" + + nTest = 0 + + for t in tests: + assert type(t) == tuple # noqa: E721 + assert len(t) == 2 + assert type(t[0]) == str # noqa: E721 + assert type(t[1]) == int # noqa: E721 + assert t[0] != "" + assert t[1] >= 0 + nTest += 1 + logging.info(templateLine.format(nTest, t[0], t[1])) # ///////////////////////////////////////////////////////////////////////////// @@ -714,6 +750,22 @@ def LOCAL__print_test_list(header: str, test_count: int, test_list: list[str]): helper__print_test_list(test_list) logging.info("") + def LOCAL__print_test_list2( + header: str, test_count: int, test_list: list[str, int] + ): + assert type(header) == str # noqa: E721 + assert type(test_count) == int # noqa: E721 + assert type(test_list) == list # noqa: E721 + assert header != "" + assert test_count >= 0 + assert len(test_list) == test_count + + LOCAL__print_line1_with_header(header) + logging.info("") + if len(test_list) > 0: + helper__print_test_list2(test_list) + logging.info("") + # fmt: off LOCAL__print_test_list( "ACHTUNG TESTS", @@ -721,13 +773,13 @@ def LOCAL__print_test_list(header: str, test_count: int, test_list: list[str]): TEST_PROCESS_STATS.AchtungTests, ) - LOCAL__print_test_list( + LOCAL__print_test_list2( "FAILED TESTS", TEST_PROCESS_STATS.cFailedTests, TEST_PROCESS_STATS.FailedTests ) - LOCAL__print_test_list( + LOCAL__print_test_list2( "XFAILED TESTS", TEST_PROCESS_STATS.cXFailedTests, TEST_PROCESS_STATS.XFailedTests, From 4c595cc5ba0048be2dc31838d8b32b9d07162a39 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 1 Apr 2025 13:36:09 +0300 Subject: [PATCH 04/10] conftest is updated [debug checks] --- tests/conftest.py | 54 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 45404d97..851e02a3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -116,65 +116,119 @@ class TEST_PROCESS_STATS: # -------------------------------------------------------------------- def incrementTotalTestCount() -> None: + assert type(__class__.cTotalTests) == int # noqa: E721 + assert __class__.cTotalTests >= 0 + __class__.cTotalTests += 1 + assert __class__.cTotalTests > 0 + # -------------------------------------------------------------------- def incrementNotExecutedTestCount() -> None: + assert type(__class__.cNotExecutedTests) == int # noqa: E721 + assert __class__.cNotExecutedTests >= 0 + __class__.cNotExecutedTests += 1 + assert __class__.cNotExecutedTests > 0 + # -------------------------------------------------------------------- def incrementExecutedTestCount() -> int: + assert type(__class__.cExecutedTests) == int # noqa: E721 + assert __class__.cExecutedTests >= 0 + __class__.cExecutedTests += 1 + + assert __class__.cExecutedTests > 0 return __class__.cExecutedTests # -------------------------------------------------------------------- def incrementPassedTestCount() -> None: + assert type(__class__.cPassedTests) == int # noqa: E721 + assert __class__.cPassedTests >= 0 + __class__.cPassedTests += 1 + assert __class__.cPassedTests > 0 + # -------------------------------------------------------------------- def incrementFailedTestCount(testID: str, errCount: int) -> None: assert type(testID) == str # noqa: E721 assert type(errCount) == int # noqa: E721 assert errCount >= 0 assert type(__class__.FailedTests) == list # noqa: E721 + assert type(__class__.cFailedTests) == int # noqa: E721 + assert __class__.cFailedTests >= 0 __class__.FailedTests.append((testID, errCount)) # raise? __class__.cFailedTests += 1 + assert len(__class__.FailedTests) > 0 + assert __class__.cFailedTests > 0 + assert len(__class__.FailedTests) == __class__.cFailedTests + # -------------------------------------------------------------------- def incrementXFailedTestCount(testID: str, errCount: int) -> None: assert type(testID) == str # noqa: E721 assert type(errCount) == int # noqa: E721 assert errCount >= 0 assert type(__class__.XFailedTests) == list # noqa: E721 + assert type(__class__.cXFailedTests) == int # noqa: E721 + assert __class__.cXFailedTests >= 0 __class__.XFailedTests.append((testID, errCount)) # raise? __class__.cXFailedTests += 1 + assert len(__class__.XFailedTests) > 0 + assert __class__.cXFailedTests > 0 + assert len(__class__.XFailedTests) == __class__.cXFailedTests + # -------------------------------------------------------------------- def incrementSkippedTestCount() -> None: + assert type(__class__.cSkippedTests) == int # noqa: E721 + assert __class__.cSkippedTests >= 0 + __class__.cSkippedTests += 1 + assert __class__.cSkippedTests > 0 + # -------------------------------------------------------------------- def incrementNotXFailedTests(testID: str) -> None: assert type(testID) == str # noqa: E721 assert type(__class__.NotXFailedTests) == list # noqa: E721 + assert type(__class__.cNotXFailedTests) == int # noqa: E721 + assert __class__.cNotXFailedTests >= 0 __class__.NotXFailedTests.append(testID) # raise? __class__.cNotXFailedTests += 1 + assert len(__class__.NotXFailedTests) > 0 + assert __class__.cNotXFailedTests > 0 + assert len(__class__.NotXFailedTests) == __class__.cNotXFailedTests + # -------------------------------------------------------------------- def incrementUnexpectedTests() -> None: + assert type(__class__.cUnexpectedTests) == int # noqa: E721 + assert __class__.cUnexpectedTests >= 0 + __class__.cUnexpectedTests += 1 + assert __class__.cUnexpectedTests > 0 + # -------------------------------------------------------------------- def incrementAchtungTestCount(testID: str) -> None: assert type(testID) == str # noqa: E721 assert type(__class__.AchtungTests) == list # noqa: E721 + assert type(__class__.cAchtungTests) == int # noqa: E721 + assert __class__.cAchtungTests >= 0 __class__.AchtungTests.append(testID) # raise? __class__.cAchtungTests += 1 + assert len(__class__.AchtungTests) > 0 + assert __class__.cAchtungTests > 0 + assert len(__class__.AchtungTests) == __class__.cAchtungTests + # ///////////////////////////////////////////////////////////////////////////// From 4e100caaf1052e82a5a043956b9e2bc5ab89fb65 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 1 Apr 2025 13:38:56 +0300 Subject: [PATCH 05/10] conftest is updated [formatting] --- tests/conftest.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 851e02a3..9809ae4d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -849,17 +849,17 @@ def LOCAL__print_test_list2( LOCAL__print_line1_with_header("SUMMARY STATISTICS") logging.info("") logging.info("[TESTS]") - logging.info(" TOTAL : {0}".format(TEST_PROCESS_STATS.cTotalTests)) - logging.info(" EXECUTED : {0}".format(TEST_PROCESS_STATS.cExecutedTests)) - logging.info(" NOT EXECUTED: {0}".format(TEST_PROCESS_STATS.cNotExecutedTests)) - logging.info(" ACHTUNG : {0}".format(TEST_PROCESS_STATS.cAchtungTests)) + logging.info(" TOTAL : {0}".format(TEST_PROCESS_STATS.cTotalTests)) + logging.info(" EXECUTED : {0}".format(TEST_PROCESS_STATS.cExecutedTests)) + logging.info(" NOT EXECUTED : {0}".format(TEST_PROCESS_STATS.cNotExecutedTests)) + logging.info(" ACHTUNG : {0}".format(TEST_PROCESS_STATS.cAchtungTests)) logging.info("") - logging.info(" PASSED : {0}".format(TEST_PROCESS_STATS.cPassedTests)) - logging.info(" FAILED : {0}".format(TEST_PROCESS_STATS.cFailedTests)) - logging.info(" XFAILED : {0}".format(TEST_PROCESS_STATS.cXFailedTests)) - logging.info(" NOT XFAILED : {0}".format(TEST_PROCESS_STATS.cNotXFailedTests)) - logging.info(" SKIPPED : {0}".format(TEST_PROCESS_STATS.cSkippedTests)) - logging.info(" UNEXPECTED : {0}".format(TEST_PROCESS_STATS.cUnexpectedTests)) + logging.info(" PASSED : {0}".format(TEST_PROCESS_STATS.cPassedTests)) + logging.info(" FAILED : {0}".format(TEST_PROCESS_STATS.cFailedTests)) + logging.info(" XFAILED : {0}".format(TEST_PROCESS_STATS.cXFailedTests)) + logging.info(" NOT XFAILED : {0}".format(TEST_PROCESS_STATS.cNotXFailedTests)) + logging.info(" SKIPPED : {0}".format(TEST_PROCESS_STATS.cSkippedTests)) + logging.info(" UNEXPECTED : {0}".format(TEST_PROCESS_STATS.cUnexpectedTests)) logging.info("") From 202e526f6d392afe1bc51d404dfc1b6f1e61f643 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 1 Apr 2025 13:41:35 +0300 Subject: [PATCH 06/10] False XFail message has 'info' status [not warning] Let's do not use error/warning for internal messages. --- tests/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9809ae4d..88b2f7a7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -446,12 +446,12 @@ def helper__makereport__call( TEST_PROCESS_STATS.incrementNotXFailedTests(testID) - warnMsg = "Test is marked as xfail" + warnMsg = "NOTE: Test is marked as xfail" if rep.wasxfail != "": warnMsg += " [" + rep.wasxfail + "]" - logging.warning(warnMsg) + logging.info(warnMsg) exitStatus = "NOT XFAILED" else: assert not hasattr(rep, "wasxfail") From b8e383d6e958bb4383400a9d5e7123f5c30cd5ca Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 1 Apr 2025 13:44:31 +0300 Subject: [PATCH 07/10] [summary] List of tests with warnings is printed. --- tests/conftest.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 88b2f7a7..36a8fedb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -106,12 +106,14 @@ class TEST_PROCESS_STATS: cXFailedTests: int = 0 cSkippedTests: int = 0 cNotXFailedTests: int = 0 + cWarningTests: int = 0 cUnexpectedTests: int = 0 cAchtungTests: int = 0 FailedTests = list[str, int]() XFailedTests = list[str, int]() NotXFailedTests = list[str]() + WarningTests = list[str, int]() AchtungTests = list[str]() # -------------------------------------------------------------------- @@ -206,6 +208,23 @@ def incrementNotXFailedTests(testID: str) -> None: assert __class__.cNotXFailedTests > 0 assert len(__class__.NotXFailedTests) == __class__.cNotXFailedTests + # -------------------------------------------------------------------- + def incrementWarningTestCount(testID: str, warningCount: int) -> None: + assert type(testID) == str # noqa: E721 + assert type(warningCount) == int # noqa: E721 + assert testID != "" + assert warningCount > 0 + assert type(__class__.WarningTests) == list # noqa: E721 + assert type(__class__.cWarningTests) == int # noqa: E721 + assert __class__.cWarningTests >= 0 + + __class__.WarningTests.append((testID, warningCount)) # raise? + __class__.cWarningTests += 1 + + assert len(__class__.WarningTests) > 0 + assert __class__.cWarningTests > 0 + assert len(__class__.WarningTests) == __class__.cWarningTests + # -------------------------------------------------------------------- def incrementUnexpectedTests() -> None: assert type(__class__.cUnexpectedTests) == int # noqa: E721 @@ -464,6 +483,10 @@ def helper__makereport__call( # [2025-03-28] It may create a useless problem in new environment. # assert False + # -------- + if item_warning_msg_count > 0: + TEST_PROCESS_STATS.incrementWarningTestCount(testID, item_warning_msg_count) + # -------- logging.info("*") logging.info("* DURATION : {0}".format(timedelta_to_human_text(testDurration))) @@ -844,6 +867,12 @@ def LOCAL__print_test_list2( TEST_PROCESS_STATS.cNotXFailedTests, TEST_PROCESS_STATS.NotXFailedTests, ) + + LOCAL__print_test_list2( + "WARNING TESTS", + TEST_PROCESS_STATS.cWarningTests, + TEST_PROCESS_STATS.WarningTests, + ) # fmt: on LOCAL__print_line1_with_header("SUMMARY STATISTICS") @@ -859,6 +888,7 @@ def LOCAL__print_test_list2( logging.info(" XFAILED : {0}".format(TEST_PROCESS_STATS.cXFailedTests)) logging.info(" NOT XFAILED : {0}".format(TEST_PROCESS_STATS.cNotXFailedTests)) logging.info(" SKIPPED : {0}".format(TEST_PROCESS_STATS.cSkippedTests)) + logging.info(" WITH WARNINGS: {0}".format(TEST_PROCESS_STATS.cWarningTests)) logging.info(" UNEXPECTED : {0}".format(TEST_PROCESS_STATS.cUnexpectedTests)) logging.info("") From f56fc94a85325066cf08e8eaf04be824855887d5 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 1 Apr 2025 14:36:03 +0300 Subject: [PATCH 08/10] TEST_PROCESS_STATS::incrementFailedTestCount is updated [debug check] --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 36a8fedb..93782723 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -157,7 +157,7 @@ def incrementPassedTestCount() -> None: def incrementFailedTestCount(testID: str, errCount: int) -> None: assert type(testID) == str # noqa: E721 assert type(errCount) == int # noqa: E721 - assert errCount >= 0 + assert errCount > 0 assert type(__class__.FailedTests) == list # noqa: E721 assert type(__class__.cFailedTests) == int # noqa: E721 assert __class__.cFailedTests >= 0 From 98d032fd048156839d4f568116727d0cf3654670 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 1 Apr 2025 14:37:45 +0300 Subject: [PATCH 09/10] [summary] total errors/warnings/duration are printed --- tests/conftest.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 93782723..ff3b3cb4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -116,6 +116,11 @@ class TEST_PROCESS_STATS: WarningTests = list[str, int]() AchtungTests = list[str]() + cTotalDuration: datetime.timedelta = datetime.timedelta() + + cTotalErrors: int = 0 + cTotalWarnings: int = 0 + # -------------------------------------------------------------------- def incrementTotalTestCount() -> None: assert type(__class__.cTotalTests) == int # noqa: E721 @@ -169,6 +174,14 @@ def incrementFailedTestCount(testID: str, errCount: int) -> None: assert __class__.cFailedTests > 0 assert len(__class__.FailedTests) == __class__.cFailedTests + # -------- + assert type(__class__.cTotalErrors) == int # noqa: E721 + assert __class__.cTotalErrors >= 0 + + __class__.cTotalErrors += errCount + + assert __class__.cTotalErrors > 0 + # -------------------------------------------------------------------- def incrementXFailedTestCount(testID: str, errCount: int) -> None: assert type(testID) == str # noqa: E721 @@ -225,6 +238,14 @@ def incrementWarningTestCount(testID: str, warningCount: int) -> None: assert __class__.cWarningTests > 0 assert len(__class__.WarningTests) == __class__.cWarningTests + # -------- + assert type(__class__.cTotalWarnings) == int # noqa: E721 + assert __class__.cTotalWarnings >= 0 + + __class__.cTotalWarnings += warningCount + + assert __class__.cTotalWarnings > 0 + # -------------------------------------------------------------------- def incrementUnexpectedTests() -> None: assert type(__class__.cUnexpectedTests) == int # noqa: E721 @@ -454,6 +475,7 @@ def helper__makereport__call( logging.error(call.excinfo.value) item_error_msg_count += 1 + assert item_error_msg_count > 0 TEST_PROCESS_STATS.incrementFailedTestCount(testID, item_error_msg_count) exitStatus = "FAILED" @@ -487,6 +509,14 @@ def helper__makereport__call( if item_warning_msg_count > 0: TEST_PROCESS_STATS.incrementWarningTestCount(testID, item_warning_msg_count) + # -------- + assert type(TEST_PROCESS_STATS.cTotalDuration) == datetime.timedelta # noqa: E721 + assert type(testDurration) == datetime.timedelta # noqa: E721 + + TEST_PROCESS_STATS.cTotalDuration += testDurration + + assert testDurration <= TEST_PROCESS_STATS.cTotalDuration + # -------- logging.info("*") logging.info("* DURATION : {0}".format(timedelta_to_human_text(testDurration))) @@ -892,6 +922,23 @@ def LOCAL__print_test_list2( logging.info(" UNEXPECTED : {0}".format(TEST_PROCESS_STATS.cUnexpectedTests)) logging.info("") + assert type(TEST_PROCESS_STATS.cTotalDuration) == datetime.timedelta # noqa: E721 + + LOCAL__print_line1_with_header("TIME") + logging.info("") + logging.info( + " TOTAL DURATION: {0}".format( + timedelta_to_human_text(TEST_PROCESS_STATS.cTotalDuration) + ) + ) + logging.info("") + + LOCAL__print_line1_with_header("TOTAL INFORMATION") + logging.info("") + logging.info(" TOTAL ERROR COUNT : {0}".format(TEST_PROCESS_STATS.cTotalErrors)) + logging.info(" TOTAL WARNING COUNT: {0}".format(TEST_PROCESS_STATS.cTotalWarnings)) + logging.info("") + # ///////////////////////////////////////////////////////////////////////////// From 282213f945a720a2f47660c0c3cbe173f370638b Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 1 Apr 2025 14:44:34 +0300 Subject: [PATCH 10/10] test_conftest.py--devel is added It is an internal code for testing a conftest.py. --- tests/test_conftest.py--devel | 80 +++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 tests/test_conftest.py--devel diff --git a/tests/test_conftest.py--devel b/tests/test_conftest.py--devel new file mode 100644 index 00000000..67c1dafe --- /dev/null +++ b/tests/test_conftest.py--devel @@ -0,0 +1,80 @@ +import pytest +import logging + + +class TestConfest: + def test_failed(self): + raise Exception("TEST EXCEPTION!") + + def test_ok(self): + pass + + @pytest.mark.skip() + def test_mark_skip__no_reason(self): + pass + + @pytest.mark.xfail() + def test_mark_xfail__no_reason(self): + raise Exception("XFAIL EXCEPTION") + + @pytest.mark.xfail() + def test_mark_xfail__no_reason___no_error(self): + pass + + @pytest.mark.skip(reason="reason") + def test_mark_skip__with_reason(self): + pass + + @pytest.mark.xfail(reason="reason") + def test_mark_xfail__with_reason(self): + raise Exception("XFAIL EXCEPTION") + + @pytest.mark.xfail(reason="reason") + def test_mark_xfail__with_reason___no_error(self): + pass + + def test_exc_skip__no_reason(self): + pytest.skip() + + def test_exc_xfail__no_reason(self): + pytest.xfail() + + def test_exc_skip__with_reason(self): + pytest.skip(reason="SKIP REASON") + + def test_exc_xfail__with_reason(self): + pytest.xfail(reason="XFAIL EXCEPTION") + + def test_log_error(self): + logging.error("IT IS A LOG ERROR!") + + def test_log_error_and_exc(self): + logging.error("IT IS A LOG ERROR!") + + raise Exception("TEST EXCEPTION!") + + def test_log_error_and_warning(self): + logging.error("IT IS A LOG ERROR!") + logging.warning("IT IS A LOG WARNING!") + logging.error("IT IS THE SECOND LOG ERROR!") + logging.warning("IT IS THE SECOND LOG WARNING!") + + @pytest.mark.xfail() + def test_log_error_and_xfail_mark_without_reason(self): + logging.error("IT IS A LOG ERROR!") + + @pytest.mark.xfail(reason="It is a reason message") + def test_log_error_and_xfail_mark_with_reason(self): + logging.error("IT IS A LOG ERROR!") + + @pytest.mark.xfail() + def test_two_log_error_and_xfail_mark_without_reason(self): + logging.error("IT IS THE FIRST LOG ERROR!") + logging.info("----------") + logging.error("IT IS THE SECOND LOG ERROR!") + + @pytest.mark.xfail(reason="It is a reason message") + def test_two_log_error_and_xfail_mark_with_reason(self): + logging.error("IT IS THE FIRST LOG ERROR!") + logging.info("----------") + logging.error("IT IS THE SECOND LOG ERROR!")