diff --git a/.travis.yml b/.travis.yml index 7557a2c..55b7afa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,11 @@ notifications: on_failure: always env: + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.8.0 PG_VERSION=17 + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.8 PG_VERSION=17 + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.9 PG_VERSION=17 + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.10 PG_VERSION=17 + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.11 PG_VERSION=17 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=16 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=15 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=14 diff --git a/Dockerfile--std2-all.tmpl b/Dockerfile--std2-all.tmpl new file mode 100644 index 0000000..10d8280 --- /dev/null +++ b/Dockerfile--std2-all.tmpl @@ -0,0 +1,96 @@ +ARG PG_VERSION +ARG PYTHON_VERSION + +# --------------------------------------------- base1 +FROM postgres:${PG_VERSION}-alpine as base1 + +# --------------------------------------------- base2_with_python-3 +FROM base1 as base2_with_python-3 +RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers + +# For pyenv +RUN apk add patch +RUN apk add git +RUN apk add xz-dev +RUN apk add zip +RUN apk add zlib-dev +RUN apk add libffi-dev +RUN apk add readline-dev +RUN apk add openssl openssl-dev +RUN apk add sqlite-dev +RUN apk add bzip2-dev + +# --------------------------------------------- base3_with_python-3.8.0 +FROM base2_with_python-3 as base3_with_python-3.8.0 +ENV PYTHON_VERSION=3.8.0 + +# --------------------------------------------- base3_with_python-3.8 +FROM base2_with_python-3 as base3_with_python-3.8 +ENV PYTHON_VERSION=3.8 + +# --------------------------------------------- base3_with_python-3.9 +FROM base2_with_python-3 as base3_with_python-3.9 +ENV PYTHON_VERSION=3.9 + +# --------------------------------------------- base3_with_python-3.10 +FROM base2_with_python-3 as base3_with_python-3.10 +ENV PYTHON_VERSION=3.10 + +# --------------------------------------------- base3_with_python-3.11 +FROM base2_with_python-3 as base3_with_python-3.11 +ENV PYTHON_VERSION=3.11 + +# --------------------------------------------- final +FROM base3_with_python-${PYTHON_VERSION} as final + +#RUN apk add --no-cache mc + +# Full version of "ps" command +RUN apk add --no-cache procps + +RUN apk add --no-cache openssh +RUN apk add --no-cache sudo + +ENV LANG=C.UTF-8 + +RUN addgroup -S sudo +RUN adduser postgres sudo + +EXPOSE 22 +RUN ssh-keygen -A + +ADD . /pg/testgres +WORKDIR /pg/testgres +RUN chown -R postgres:postgres /pg + +# It allows to use sudo without password +RUN sh -c "echo \"postgres ALL=(ALL:ALL) NOPASSWD:ALL\"">>/etc/sudoers + +# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD +RUN sh -c "echo "postgres:*" | chpasswd -e" + +USER postgres + +RUN curl https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash + +RUN ~/.pyenv/bin/pyenv install ${PYTHON_VERSION} + +# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD +RUN chmod 700 ~/ + +RUN mkdir -p ~/.ssh +#RUN chmod 700 ~/.ssh + +ENTRYPOINT sh -c " \ +set -eux; \ +echo HELLO FROM ENTRYPOINT; \ +echo HOME DIR IS [`realpath ~/`]; \ +ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ +chmod 600 ~/.ssh/authorized_keys; \ +ls -la ~/.ssh/; \ +sudo /usr/sbin/sshd; \ +ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ +ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ +export PATH=\"~/.pyenv/bin:$PATH\"; \ +TEST_FILTER=\"\" bash run_tests2.sh;" diff --git a/run_tests2.sh b/run_tests2.sh new file mode 100755 index 0000000..173b19d --- /dev/null +++ b/run_tests2.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +# Copyright (c) 2017-2025 Postgres Professional + +set -eux + +eval "$(pyenv init -)" +eval "$(pyenv virtualenv-init -)" + +pyenv virtualenv --force ${PYTHON_VERSION} cur +pyenv activate cur + +if [ -z ${TEST_FILTER+x} ]; \ +then export TEST_FILTER="TestTestgresLocal or (TestTestgresCommon and (not remote))"; \ +fi + +# fail early +echo check that pg_config is in PATH +command -v pg_config + +# prepare python environment +VENV_PATH="/tmp/testgres_venv" +rm -rf $VENV_PATH +python -m venv "${VENV_PATH}" +export VIRTUAL_ENV_DISABLE_PROMPT=1 +source "${VENV_PATH}/bin/activate" +pip install coverage flake8 psutil Sphinx pytest pytest-xdist psycopg2 six psutil + +# install testgres' dependencies +export PYTHONPATH=$(pwd) +# $PIP install . + +# test code quality +flake8 . + + +# remove existing coverage file +export COVERAGE_FILE=.coverage +rm -f $COVERAGE_FILE + + +# run tests (PATH) +time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" + + +# run tests (PG_BIN) +PG_BIN=$(pg_config --bindir) \ +time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" + + +# run tests (PG_CONFIG) +PG_CONFIG=$(pg_config --bindir)/pg_config \ +time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" + + +# show coverage +coverage report + +# build documentation +cd docs +make html +cd .. + +# attempt to fix codecov +set +eux + +# send coverage stats to Codecov +bash <(curl -s https://codecov.io/bash) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 25d02f3..33b61ac 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -5,6 +5,7 @@ import tempfile import io import logging +import typing from ..exceptions import ExecUtilException from ..exceptions import InvalidOperationException @@ -669,8 +670,8 @@ def _is_port_free__process_1(error: str) -> bool: return True @staticmethod - def _make_exec_env_list() -> list[str]: - result = list[str]() + def _make_exec_env_list() -> typing.List[str]: + result: typing.List[str] = list() for envvar in os.environ.items(): if not __class__._does_put_envvar_into_exec_cmd(envvar[0]): continue diff --git a/testgres/port_manager.py b/testgres/port_manager.py index 164661e..e253047 100644 --- a/testgres/port_manager.py +++ b/testgres/port_manager.py @@ -6,6 +6,7 @@ import threading import random +import typing class PortManager: @@ -50,16 +51,16 @@ class PortManager__Generic(PortManager): _os_ops: OsOperations _guard: object # TODO: is there better to use bitmap fot _available_ports? - _available_ports: set[int] - _reserved_ports: set[int] + _available_ports: typing.Set[int] + _reserved_ports: typing.Set[int] def __init__(self, os_ops: OsOperations): assert os_ops is not None assert isinstance(os_ops, OsOperations) self._os_ops = os_ops self._guard = threading.Lock() - self._available_ports = set[int](range(1024, 65535)) - self._reserved_ports = set[int]() + self._available_ports: typing.Set[int] = set(range(1024, 65535)) + self._reserved_ports: typing.Set[int] = set() def reserve_port(self) -> int: assert self._guard is not None diff --git a/tests/conftest.py b/tests/conftest.py index ff3b3cb..6f2f9e4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,6 +26,10 @@ class TestConfigPropNames: TEST_CFG__LOG_DIR = "TEST_CFG__LOG_DIR" +# ///////////////////////////////////////////////////////////////////////////// + +T_TUPLE__str_int = typing.Tuple[str, int] + # ///////////////////////////////////////////////////////////////////////////// # TestStartupData__Helper @@ -110,11 +114,11 @@ class TEST_PROCESS_STATS: cUnexpectedTests: int = 0 cAchtungTests: int = 0 - FailedTests = list[str, int]() - XFailedTests = list[str, int]() - NotXFailedTests = list[str]() - WarningTests = list[str, int]() - AchtungTests = list[str]() + FailedTests: typing.List[T_TUPLE__str_int] = list() + XFailedTests: typing.List[T_TUPLE__str_int] = list() + NotXFailedTests: typing.List[str] = list() + WarningTests: typing.List[T_TUPLE__str_int] = list() + AchtungTests: typing.List[str] = list() cTotalDuration: datetime.timedelta = datetime.timedelta() @@ -769,7 +773,7 @@ def helper__calc_W(n: int) -> int: # ------------------------------------------------------------------------ -def helper__print_test_list(tests: list[str]) -> None: +def helper__print_test_list(tests: typing.List[str]) -> None: assert type(tests) == list # noqa: E721 assert helper__calc_W(9) == 1 @@ -796,7 +800,7 @@ def helper__print_test_list(tests: list[str]) -> None: # ------------------------------------------------------------------------ -def helper__print_test_list2(tests: list[str, int]) -> None: +def helper__print_test_list2(tests: typing.List[T_TUPLE__str_int]) -> None: assert type(tests) == list # noqa: E721 assert helper__calc_W(9) == 1 @@ -843,7 +847,7 @@ def LOCAL__print_line1_with_header(header: str): assert header != "" logging.info(C_LINE1 + " [" + header + "]") - def LOCAL__print_test_list(header: str, test_count: int, test_list: list[str]): + def LOCAL__print_test_list(header: str, test_count: int, test_list: typing.List[str]): assert type(header) == str # noqa: E721 assert type(test_count) == int # noqa: E721 assert type(test_list) == list # noqa: E721 @@ -858,7 +862,7 @@ def LOCAL__print_test_list(header: str, test_count: int, test_list: list[str]): logging.info("") def LOCAL__print_test_list2( - header: str, test_count: int, test_list: list[str, int] + header: str, test_count: int, test_list: typing.List[T_TUPLE__str_int] ): assert type(header) == str # noqa: E721 assert type(test_count) == int # noqa: E721 diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py index 7d18377..ecfff5b 100644 --- a/tests/test_os_ops_common.py +++ b/tests/test_os_ops_common.py @@ -12,13 +12,14 @@ import logging import socket import threading +import typing from ..testgres import InvalidOperationException from ..testgres import ExecUtilException class TestOsOpsCommon: - sm_os_ops_descrs: list[OsOpsDescr] = [ + sm_os_ops_descrs: typing.List[OsOpsDescr] = [ OsOpsDescrs.sm_local_os_ops_descr, OsOpsDescrs.sm_remote_os_ops_descr ] diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index c384dfb..e1252de 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -56,7 +56,7 @@ def removing(os_ops: OsOperations, f): class TestTestgresCommon: - sm_node_svcs: list[PostgresNodeService] = [ + sm_node_svcs: typing.List[PostgresNodeService] = [ PostgresNodeServices.sm_local, PostgresNodeServices.sm_local2, PostgresNodeServices.sm_remote, @@ -315,8 +315,8 @@ def test_child_pids(self, node_svc: PostgresNodeService): def LOCAL__test_auxiliary_pids( node: PostgresNode, - expectedTypes: list[ProcessType] - ) -> list[ProcessType]: + expectedTypes: typing.List[ProcessType] + ) -> typing.List[ProcessType]: # returns list of the absence processes assert node is not None assert type(node) == PostgresNode # noqa: E721 @@ -327,7 +327,7 @@ def LOCAL__test_auxiliary_pids( assert pids is not None # noqa: E721 assert type(pids) == dict # noqa: E721 - result = list[ProcessType]() + result: typing.List[ProcessType] = list() for ptype in expectedTypes: if not (ptype in pids): result.append(ptype) @@ -335,7 +335,7 @@ def LOCAL__test_auxiliary_pids( def LOCAL__check_auxiliary_pids__multiple_attempts( node: PostgresNode, - expectedTypes: list[ProcessType]): + expectedTypes: typing.List[ProcessType]): assert node is not None assert type(node) == PostgresNode # noqa: E721 assert expectedTypes is not None diff --git a/tests/test_utils.py b/tests/test_utils.py index d4a4c9a..c05bd2f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -7,10 +7,11 @@ from ..testgres import scoped_config import pytest +import typing class TestUtils: - sm_os_ops_descrs: list[OsOpsDescr] = [ + sm_os_ops_descrs: typing.List[OsOpsDescr] = [ OsOpsDescrs.sm_local_os_ops_descr, OsOpsDescrs.sm_remote_os_ops_descr ]