diff options
Diffstat (limited to 'util/testrunner/tests/tst_testrunner.py')
| -rwxr-xr-x | util/testrunner/tests/tst_testrunner.py | 401 |
1 files changed, 0 insertions, 401 deletions
diff --git a/util/testrunner/tests/tst_testrunner.py b/util/testrunner/tests/tst_testrunner.py deleted file mode 100755 index 4a3d92e167b..00000000000 --- a/util/testrunner/tests/tst_testrunner.py +++ /dev/null @@ -1,401 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (C) 2021 The Qt Company Ltd. -# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 - - -import sys -import os -import re -import glob -import subprocess - -from subprocess import STDOUT, PIPE -from tempfile import TemporaryDirectory, mkstemp - -MY_NAME = os.path.basename(__file__) -my_dir = os.path.dirname(__file__) -testrunner = os.path.join(my_dir, "..", "qt-testrunner.py") -mock_test = os.path.join(my_dir, "qt_mock_test.py") -xml_log_template = os.path.join(my_dir, "qt_mock_test-log.xml") - -with open(xml_log_template) as f: - XML_TEMPLATE = f.read() - - -import unittest - -def setUpModule(): - global TEMPDIR - TEMPDIR = TemporaryDirectory(prefix="tst_testrunner-") - - filename = os.path.join(TEMPDIR.name, "file_1") - print("setUpModule(): setting up temporary directory and env var" - " QT_MOCK_TEST_STATE_FILE=" + filename + " and" - " QT_MOCK_TEST_XML_TEMPLATE_FILE=" + xml_log_template) - - os.environ["QT_MOCK_TEST_STATE_FILE"] = filename - os.environ["QT_MOCK_TEST_XML_TEMPLATE_FILE"] = xml_log_template - -def tearDownModule(): - print("\ntearDownModule(): Cleaning up temporary directory:", - TEMPDIR.name) - del os.environ["QT_MOCK_TEST_STATE_FILE"] - TEMPDIR.cleanup() - - -# Helper to run a command and always capture output -def run(*args, **kwargs): - if DEBUG: - print("Running: ", args, flush=True) - proc = subprocess.run(*args, stdout=PIPE, stderr=STDOUT, **kwargs) - if DEBUG and proc.stdout: - print(proc.stdout.decode(), flush=True) - return proc - -# Helper to run qt-testrunner.py with proper testing arguments. -def run_testrunner(xml_filename=None, testrunner_args=None, - wrapper_script=None, wrapper_args=None, - qttest_args=None, env=None): - - args = [ testrunner ] - if xml_filename: - args += [ "--parse-xml-testlog", xml_filename ] - if testrunner_args: - args += testrunner_args - - if wrapper_script: - args += [ wrapper_script ] - if wrapper_args: - args += wrapper_args - - args += [ mock_test ] - if qttest_args: - args += qttest_args - - return run(args, env=env) - -# Write the XML_TEMPLATE to filename, replacing the templated results. -def write_xml_log(filename, failure=None): - data = XML_TEMPLATE - # Replace what was asked to fail with "fail" - if type(failure) in (list, tuple): - for template in failure: - data = data.replace("{{"+template+"_result}}", "fail") - elif type(failure) is str: - data = data.replace("{{"+failure+"_result}}", "fail") - # Replace the rest with "pass" - data = re.sub(r"{{[^}]+}}", "pass", data) - with open(filename, "w") as f: - f.write(data) - - -# Test that qt_mock_test.py behaves well. This is necessary to properly -# test qt-testrunner. -class Test_qt_mock_test(unittest.TestCase): - def setUp(self): - state_file = os.environ["QT_MOCK_TEST_STATE_FILE"] - if os.path.exists(state_file): - os.remove(state_file) - def test_always_pass(self): - proc = run([mock_test, "always_pass"]) - self.assertEqual(proc.returncode, 0) - def test_always_fail(self): - proc = run([mock_test, "always_fail"]) - self.assertEqual(proc.returncode, 1) - def test_fail_then_pass_2(self): - proc = run([mock_test, "fail_then_pass:2"]) - self.assertEqual(proc.returncode, 1) - proc = run([mock_test, "fail_then_pass:2"]) - self.assertEqual(proc.returncode, 1) - proc = run([mock_test, "fail_then_pass:2"]) - self.assertEqual(proc.returncode, 0) - def test_fail_then_pass_1(self): - proc = run([mock_test, "fail_then_pass:1"]) - self.assertEqual(proc.returncode, 1) - proc = run([mock_test, "fail_then_pass:1"]) - self.assertEqual(proc.returncode, 0) - def test_fail_then_pass_many_tests(self): - proc = run([mock_test, "fail_then_pass:1"]) - self.assertEqual(proc.returncode, 1) - proc = run([mock_test, "fail_then_pass:2"]) - self.assertEqual(proc.returncode, 1) - proc = run([mock_test, "fail_then_pass:1"]) - self.assertEqual(proc.returncode, 0) - proc = run([mock_test, "fail_then_pass:2"]) - self.assertEqual(proc.returncode, 1) - proc = run([mock_test, "fail_then_pass:2"]) - self.assertEqual(proc.returncode, 0) - def test_xml_file_is_written(self): - filename = os.path.join(TEMPDIR.name, "testlog.xml") - proc = run([mock_test, "-o", filename+",xml"]) - self.assertEqual(proc.returncode, 0) - self.assertTrue(os.path.exists(filename)) - self.assertGreater(os.path.getsize(filename), 0) - os.remove(filename) - -# Test regular invocations of qt-testrunner. -class Test_testrunner(unittest.TestCase): - def setUp(self): - state_file = os.environ["QT_MOCK_TEST_STATE_FILE"] - if os.path.exists(state_file): - os.remove(state_file) - # The mock_test honors only the XML output arguments, the rest are ignored. - old_logfiles = glob.glob(os.path.basename(mock_test) + "*.xml", - root_dir=TEMPDIR.name) - for fname in old_logfiles: - os.remove(os.path.join(TEMPDIR.name, fname)) - self.env = dict() - self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] = os.environ["QT_MOCK_TEST_XML_TEMPLATE_FILE"] - self.env["QT_MOCK_TEST_STATE_FILE"] = state_file - self.testrunner_args = [ "--log-dir", TEMPDIR.name ] - def prepare_env(self, run_list=None): - if run_list is not None: - self.env['QT_MOCK_TEST_RUN_LIST'] = ",".join(run_list) - def run2(self): - return run_testrunner(testrunner_args=self.testrunner_args, env=self.env) - def test_simple_invocation(self): - # All tests pass. - proc = self.run2() - self.assertEqual(proc.returncode, 0) - def test_always_pass(self): - self.prepare_env(run_list=["always_pass"]) - proc = self.run2() - self.assertEqual(proc.returncode, 0) - def test_output_files_are_generated(self): - proc = self.run2() - xml_output_files = glob.glob(os.path.basename(mock_test) + "-*[0-9].xml", - root_dir=TEMPDIR.name) - if DEBUG: - print("Output files found: ", - xml_output_files) - self.assertEqual(len(xml_output_files), 1) - def test_always_fail(self): - self.prepare_env(run_list=["always_fail"]) - proc = self.run2() - # TODO verify that re-runs==max_repeats - self.assertEqual(proc.returncode, 2) - def test_flaky_pass_1(self): - self.prepare_env(run_list=["always_pass,fail_then_pass:1"]) - proc = self.run2() - self.assertEqual(proc.returncode, 0) - def test_flaky_pass_5(self): - self.prepare_env(run_list=["always_pass,fail_then_pass:1,fail_then_pass:5"]) - proc = self.run2() - self.assertEqual(proc.returncode, 0) - def test_flaky_fail(self): - self.prepare_env(run_list=["always_pass,fail_then_pass:6"]) - proc = self.run2() - self.assertEqual(proc.returncode, 2) - def test_flaky_pass_fail(self): - self.prepare_env(run_list=["always_pass,fail_then_pass:1,fail_then_pass:6"]) - proc = self.run2() - # TODO verify that one func was re-run and passed but the other failed. - self.assertEqual(proc.returncode, 2) - def test_initTestCase_fail_crash(self): - self.prepare_env(run_list=["initTestCase,always_pass"]) - proc = self.run2() - self.assertEqual(proc.returncode, 3) - - # If no XML file is found by qt-testrunner, it is usually considered a - # CRASH and the whole test is re-run. Even when the return code is zero. - # It is a PASS only if the test is not capable of XML output (see no_extra_args, TODO test it). - def test_no_xml_log_written_pass_crash(self): - del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] - self.prepare_env(run_list=["always_pass"]) - proc = self.run2() - self.assertEqual(proc.returncode, 3) - # On the 2nd iteration of the full test, both of the tests pass. - # Still it's a CRASH because no XML file was found. - def test_no_xml_log_written_fail_then_pass_crash(self): - del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] - self.prepare_env(run_list=["always_pass,fail_then_pass:1"]) - proc = self.run2() - # TODO verify that the whole test has run twice. - self.assertEqual(proc.returncode, 3) - # Even after 2 iterations of the full test we still get failures but no XML file, - # and this is considered a CRASH. - def test_no_xml_log_written_crash(self): - del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] - self.prepare_env(run_list=["fail_then_pass:2"]) - proc = self.run2() - self.assertEqual(proc.returncode, 3) - - # If a test returns success but XML contains failures, it's a CRASH. - def test_wrong_xml_log_written_1_crash(self): - logfile = os.path.join(TEMPDIR.name, os.path.basename(mock_test) + ".xml") - write_xml_log(logfile, failure="always_fail") - del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] - self.prepare_env(run_list=["always_pass"]) - proc = self.run2() - self.assertEqual(proc.returncode, 3) - # If a test returns failure but XML contains only pass, it's a CRASH. - def test_wrong_xml_log_written_2_crash(self): - logfile = os.path.join(TEMPDIR.name, os.path.basename(mock_test) + ".xml") - write_xml_log(logfile) - del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] - self.prepare_env(run_list=["always_fail"]) - proc = self.run2() - self.assertEqual(proc.returncode, 3) - - def create_wrapper(self, filename, content=None): - if not content: - content='exec "$@"' - filename = os.path.join(TEMPDIR.name, filename) - # if os.path.exists(filename): - # os.remove(filename) - with open(filename, "w") as f: - f.write(f'#!/bin/sh\n{content}\n') - self.wrapper_script = f.name - os.chmod(self.wrapper_script, 0o700) - - # Test that qt-testrunner detects the correct executable name even if we - # use a special wrapper script, and that it uses that in the XML log filename. - def test_wrapper(self): - self.create_wrapper("coin_vxworks_qemu_runner.sh") - proc = run_testrunner(wrapper_script=self.wrapper_script, - testrunner_args=["--log-dir",TEMPDIR.name], - env=self.env) - self.assertEqual(proc.returncode, 0) - xml_output_files = glob.glob(os.path.basename(mock_test) + "-*[0-9].xml", - root_dir=TEMPDIR.name) - if DEBUG: - print("XML output files found: ", xml_output_files) - self.assertEqual(len(xml_output_files), 1) - - # The "androidtestrunner" wrapper is special. It expects the QTest arguments after "--". - # So our mock androidtestrunner wrapper ignores everything before "--" - # and executes our hardcoded mock_test with the arguments that follow. - def create_mock_anroidtestrunner_wrapper(self): - self.create_wrapper("androidtestrunner", content= - 'while [ "$1" != "--" ]; do shift; done; shift; exec {} "$@"'.format(mock_test)) - - def test_androidtestrunner_with_aab(self): - self.create_mock_anroidtestrunner_wrapper() - # Copied verbatim from our CI logs. The only relevant option is --aab. - androidtestrunner_args= ['--path', '/home/qt/work/qt/qtdeclarative_standalone_tests/tests/auto/quickcontrols/qquickpopup/android-build-tst_qquickpopup', '--adb', '/opt/android/sdk/platform-tools/adb', '--skip-install-root', '--ndk-stack', '/opt/android/android-ndk-r27c/ndk-stack', '--manifest', '/home/qt/work/qt/qtdeclarative_standalone_tests/tests/auto/quickcontrols/qquickpopup/android-build-tst_qquickpopup/app/AndroidManifest.xml', '--make', '"/opt/cmake-3.30.5/bin/cmake" --build /home/qt/work/qt/qtdeclarative_standalone_tests --target tst_qquickpopup_make_aab', '--aab', '/home/qt/work/qt/qtdeclarative_standalone_tests/tests/auto/quickcontrols/qquickpopup/android-build-tst_qquickpopup/tst_qquickpopup.aab', '--bundletool', '/opt/bundletool/bundletool', '--timeout', '1425'] - # In COIN CI, TESTRUNNER="qt-testrunner.py --". That's why we append "--". - proc = run_testrunner(testrunner_args=["--log-dir", TEMPDIR.name, "--"], - wrapper_script=self.wrapper_script, - wrapper_args=androidtestrunner_args, - env=self.env) - self.assertEqual(proc.returncode, 0) - xml_output_files = glob.glob("tst_qquickpopup-*[0-9].xml", - root_dir=TEMPDIR.name) - if DEBUG: - print("XML output files found: ", xml_output_files) - self.assertEqual(len(xml_output_files), 1) - # similar to above but with "--apk" - def test_androidtestrunner_with_apk(self): - self.create_mock_anroidtestrunner_wrapper() - androidtestrunner_args= ['--blah', '--apk', '/whatever/waza.apk', 'blue'] - proc = run_testrunner(testrunner_args=["--log-dir", TEMPDIR.name, "--"], - wrapper_script=self.wrapper_script, - wrapper_args=androidtestrunner_args, - env=self.env) - self.assertEqual(proc.returncode, 0) - xml_output_files = glob.glob("waza-*[0-9].xml", - root_dir=TEMPDIR.name) - if DEBUG: - print("XML output files found: ", xml_output_files) - self.assertEqual(len(xml_output_files), 1) - # similar to above but with neither "--apk" nor "--aab". qt-testrunner throws error. - def test_androidtestrunner_fail_to_detect_filename(self): - self.create_mock_anroidtestrunner_wrapper() - androidtestrunner_args= ['--blah', '--argh', '/whatever/waza.apk', 'waza.aab'] - proc = run_testrunner(testrunner_args=["--log-dir", TEMPDIR.name, "--"], - wrapper_script=self.wrapper_script, - wrapper_args=androidtestrunner_args, - env=self.env) - self.assertEqual(proc.returncode, 1) - xml_output_files = glob.glob("waza-*[0-9].xml", - root_dir=TEMPDIR.name) - if DEBUG: - print("XML output files found: ", xml_output_files) - self.assertEqual(len(xml_output_files), 0) - - -# Test qt-testrunner script with an existing XML log file: -# qt-testrunner.py qt_mock_test.py --parse-xml-testlog file.xml -# qt-testrunner should repeat the testcases that are logged as -# failures and fail or pass depending on how the testcases behave. -# Different XML files are generated for the following test cases. -# + No failure logged. qt-testrunner should exit(0) -# + The "always_pass" test has failed. qt-testrunner should exit(0). -# + The "always_fail" test has failed. qt-testrunner should exit(2). -# + The "always_crash" test has failed. qt-testrunner should exit(2). -# + The "fail_then_pass:2" test failed. qt-testrunner should exit(0). -# + The "fail_then_pass:5" test failed. qt-testrunner should exit(2). -# + The "initTestCase" failed which is listed as NO_RERUN thus -# qt-testrunner should exit(3). -class Test_testrunner_with_xml_logfile(unittest.TestCase): - # Runs before every single test function, creating a unique temp file. - def setUp(self): - (_handle, self.xml_file) = mkstemp( - suffix=".xml", prefix="qt_mock_test-log-", - dir=TEMPDIR.name) - if os.path.exists(os.environ["QT_MOCK_TEST_STATE_FILE"]): - os.remove(os.environ["QT_MOCK_TEST_STATE_FILE"]) - def tearDown(self): - os.remove(self.xml_file) - del self.xml_file - - def test_no_failure(self): - write_xml_log(self.xml_file, failure=None) - proc = run_testrunner(self.xml_file) - self.assertEqual(proc.returncode, 0) - def test_always_pass_failed(self): - write_xml_log(self.xml_file, failure="always_pass") - proc = run_testrunner(self.xml_file) - self.assertEqual(proc.returncode, 0) - def test_always_pass_failed_max_repeats_0(self): - write_xml_log(self.xml_file, failure="always_pass") - proc = run_testrunner(self.xml_file, - testrunner_args=["--max-repeats", "0"]) - self.assertEqual(proc.returncode, 2) - def test_always_fail_failed(self): - write_xml_log(self.xml_file, failure="always_fail") - proc = run_testrunner(self.xml_file) - self.assertEqual(proc.returncode, 2) - # Assert that one of the re-runs was in verbose mode - matches = re.findall("VERBOSE RUN", - proc.stdout.decode()) - self.assertEqual(len(matches), 1) - # Assert that the environment was altered too - self.assertIn("QT_LOGGING_RULES", proc.stdout.decode()) - def test_always_crash_failed(self): - write_xml_log(self.xml_file, failure="always_crash") - proc = run_testrunner(self.xml_file) - self.assertEqual(proc.returncode, 2) - def test_fail_then_pass_2_failed(self): - write_xml_log(self.xml_file, failure="fail_then_pass:2") - proc = run_testrunner(self.xml_file) - self.assertEqual(proc.returncode, 0) - def test_fail_then_pass_5_failed(self): - write_xml_log(self.xml_file, failure="fail_then_pass:5") - proc = run_testrunner(self.xml_file) - self.assertEqual(proc.returncode, 2) - def test_with_two_failures(self): - write_xml_log(self.xml_file, - failure=["always_pass", "fail_then_pass:2"]) - proc = run_testrunner(self.xml_file) - self.assertEqual(proc.returncode, 0) - # Check that test output is properly interleaved with qt-testrunner's logging. - matches = re.findall(r"(PASS|FAIL!).*\n.*Test process exited with code", - proc.stdout.decode()) - self.assertEqual(len(matches), 4) - def test_initTestCase_fail_crash(self): - write_xml_log(self.xml_file, failure="initTestCase") - proc = run_testrunner(self.xml_file) - self.assertEqual(proc.returncode, 3) - - -if __name__ == "__main__": - - DEBUG = False - if "--debug" in sys.argv: - sys.argv.remove("--debug") - DEBUG = True - - # We set failfast=True as we do not want the test suite to continue if the - # tests of qt_mock_test failed. The next ones depend on it. - unittest.main(failfast=True) |