diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..800d486 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,35 @@ +name: osrf_pycommon-ci + +on: + push: + branches: [master] + pull_request: + +jobs: + build: + strategy: + matrix: + os: [macos-latest, ubuntu-22.04, windows-latest] + python: ['3.8', '3.9', '3.10', '3.11', '3.12'] + include: + - os: ubuntu-20.04 + python: '3.6' + name: osrf_pycommon tests + runs-on: ${{matrix.os}} + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{matrix.python}} + uses: actions/setup-python@v5 + with: + python-version: ${{matrix.python}} + - name: Install dependencies + run: | + python -m pip install -U -e .[test] pytest-cov + - name: Run tests + run: | + python -m pytest tests --cov=osrf_pycommon + - name: Upload coverage + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 5fb08a2..0000000 --- a/.travis.yml +++ /dev/null @@ -1,15 +0,0 @@ -language: python -python: - - "2.7" - - "3.5" - - "3.6" - - "3.7" - - "3.8" -install: - - pip install coverage nose flake8 mock - - pip install git+https://github.com/PyCQA/flake8-import-order.git -script: - - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then COVER_INCLUSIVE=--cover-inclusive; fi - - PYTHONASYNCIODEBUG=1 python setup.py nosetests -s --with-coverage $COVER_INCLUSIVE -notifications: - email: false diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fcab3ec..07e4259 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,111 @@ +2.1.6 (2025-03-25) +------------------ +* Merge pull request `#103 `_ from christophebedard/christophebedard/fix-typo-on-each-verb +* Contributors: Christophe Bedard + +2.1.5 (2024-12-18) +------------------ +* Align stdeb dependencies with setup.py (`#101 `_) + Follow-up to 4b2f3a8e4969f33dced1dc2db2296230e7a55b1d +* Add '+upstream' suffix to published deb version (`#102 `_) + Using a debian version suffix which falls late alphabetically appears to + give our packages preference by apt. If a user enables a repository + which distributes packages created by OSRF or ROS, it is likely that + they wish to use these packages instead of the ones packaged by their + platform. +* Upload coverage results to codecov (`#100 `_) +* Update ci.yaml (`#96 `_) + fix node.js <20 deprecation + Co-authored-by: Scott K Logan +* Updated python version (`#97 `_) + Python version 3.7 is no longer supported as of June 27, 2023 + Co-authored-by: Scott K Logan +* Resolve outstanding resource warnings when running tests (`#99 `_) +* Update deb platforms for release (`#95 `_) + Added: + * Ubuntu Noble (24.04 LTS pre-release) + * Debian Trixie (testing) + Dropped: + * Debian Bullseye (oldstable) + Retained: + * Debian Bookworm (stable) + * Ubuntu Focal (20.04 LTS) + * Ubuntu Jammy (22.04 LTS) +* Remove CODEOWNERS. (`#98 `_) + It is out of date and no longer serving its intended purpose. +* Contributors: Chris Lalancette, Scott K Logan, Steven! Ragnarök, mosfet80 + +2.1.4 (2023-08-21) +------------------ +* Catch all of the spurious warnings from get_event_loop. (`#94 `_) +* Contributors: Chris Lalancette + +2.1.3 (2023-07-11) +------------------ +* Add bookworm as a python3 target (`#91 `_) +* Suppress warning for specifically handled behavior (`#87 `_) +* Update supported platforms (`#93 `_) +* Add GitHub Actions CI workflow (`#88 `_) +* Contributors: Scott K Logan, Tully Foote + +2.1.2 (2023-02-14) +------------------ +* [master] Update maintainers - 2022-11-07 (`#89 `_) +* Contributors: Audrow Nash + +2.1.1 (2022-11-07) +------------------ +* Declare test dependencies in [test] extra (`#86 `_) +* Contributors: Scott K Logan + +2.1.0 (2022-05-10) +------------------ + +2.0.2 (2022-04-08) +------------------ +* Fix an importlib_metadata warning with Python 3.10. (`#84 `_) +* Contributors: Chris Lalancette + +2.0.1 (2022-02-14) +------------------ +* Don't release 2.x / master on Debian Buster. (`#83 `_) + Debian Buster is on Python 3.7: https://packages.debian.org/buster/python3 +* Stop using mock in favor of unittest.mock. (`#74 `_) + Mock has been deprecated since Python 3.3; see + https://pypi.org/project/mock/ . The recommended replacement + is unittest.mock, which seems to be a drop-in replacement. + Co-authored-by: William Woodall +* Fix dependencies (`#81 `_) + * Remove obsolete setuptools from install_requires + Now that pkg_resources are no longer used, there is no need to depend + on setuptools at runtime. + * Fix version-conditional dependency on importlib-metadata + Use version markers to depend on importlib-metadata correctly. Explicit + conditions mean that wheels built with setup.py will either have the dep + or not depending on what Python version they're built with, rather than + what version they're installed on. +* fix whitespace and date in changelog heading +* Contributors: Chris Lalancette, Michał Górny, Steven! Ragnarök, William Woodall + +2.0.0 (2022-02-01) +------------------ +* Replace the use of ``pkg_resources`` with the more modern ``importlib-metadata``. (`#66 `_) + * Note this means that from now on you can only release on >= Ubuntu focal as that was when ``python3-importlib-metadata`` was introduced. + * Used the ``1.0.x`` branch if you need an ealier version that still uses ``pkg_resources``. + Co-authored-by: William Woodall +* Contributors: Chris Lalancette + +1.0.1 (2022-01-20) +------------------ +* Update release distributions. (`#78 `_) +* Contributors: Steven! Ragnarök + +1.0.0 (2021-01-25) +------------------ +* Added missing conflict rules in stdeb.cfg. +* Removed Python 2 support. +* Contributors: Chris Lalancette, Timon Engelke + 0.2.1 (2021-01-25) ------------------ * Fix osrf.py_common.process_utils.get_loop() implementation (`#70 `_) diff --git a/README.md b/README.md index fd46c4c..20f1695 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,11 @@ osrf_pycommon ============= -Commonly needed Python modules, used by Python software developed at OSRF +Commonly needed Python modules, used by Python software developed at OSRF. + +Branches +======== + +If you are releasing (using ``stdeb`` or on the ROS buildfarm) for any Ubuntu < ``focal``, or for any OS that doesn't have a key for ``python3-importlib-metadata``, then you need to use the ``1.0.x`` branch, or the latest ``1.`` branch, because starting with ``2.0.0``, that dependency will be required. + +If you are using Python 2, then you should use the ``python2`` branch. diff --git a/docs/index.rst b/docs/index.rst index 3fc4db0..86b0ddb 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -5,8 +5,8 @@ Things like ansi terminal coloring, capturing colored output from programs using subprocess, or even a simple logging system which provides some nice functionality over the built-in Python logging system. The functionality provided here should be generic enough to be reused in arbitrary scenarios and should avoid bringing in dependencies which are not part of the standard Python library. -Where possible Windows and Linux/OS X should be supported, and where it cannot it should be gracefully degrading. -Code should be pure Python as well as Python 2 and Python 3 bilingual. +Where possible Windows, Linux, and macOS should be supported, and where it cannot it should be gracefully degrading. +Code should be pure Python 3. Contents: @@ -64,13 +64,13 @@ That will "uninstall" the hooks into the ``PYTHONPATH`` which point to your sour Testing ------- -In order to run the tests you will need to install `nosetests `_, `flake8 `_, and `Mock `_. +In order to run the tests you will need to install `flake8 `_. -Once you have installed those, then run ``nosetest`` in the root of the ``osrf_pycommon`` source directory: +Once you have installed those, then run ``unittest``: .. code-block:: bash - $ nosetests + $ python3 -m unittest discover -v tests Building the Documentation -------------------------- diff --git a/docs/process_utils.rst b/docs/process_utils.rst index 40b6edc..3ee9e8a 100644 --- a/docs/process_utils.rst +++ b/docs/process_utils.rst @@ -12,14 +12,14 @@ These are the main sections of this module: Asynchronous Process Utilities ------------------------------ -There is a function and class which can be used together with your custom `Tollius `_ or `asyncio `_ run loop. +There is a function and class which can be used together with your custom `asyncio `_ run loop. The :py:func:`osrf_pycommon.process_utils.async_execute_process` function is a `coroutine `_ which allows you to run a process and get the output back bit by bit in real-time, either with stdout and stderr separated or combined. This function also allows you to emulate the terminal using a pty simply by toggling a flag in the parameters. Along side this coroutine is a `Protocol `_ class, :py:class:`osrf_pycommon.process_utils.AsyncSubprocessProtocol`, from which you can inherit in order to customize how the yielded output is handled. -Because this coroutine is built on the ``trollius``/``asyncio`` framework's subprocess functions, it is portable and should behave the same on all major OS's. (including on Windows where an IOCP implementation is used) +Because this coroutine is built on the ``asyncio`` framework's subprocess functions, it is portable and should behave the same on all major OS's. (including on Windows where an IOCP implementation is used) .. autofunction:: osrf_pycommon.process_utils.async_execute_process @@ -33,9 +33,11 @@ In addtion to these functions, there is a utility function for getting the corre Treatment of File Descriptors ----------------------------- -Unlike ``subprocess.Popen``, all of the ``process_utils`` functions behave the same way on Python versions 2.7 through 3.4, and they do not close `inheritable `. file descriptors before starting subprocesses. This is equivalent to passing ``close_fds=False`` to ``subprocess.Popen`` on all Python versions. +Like Python 3.4's ``subprocess.Popen`` (and newer versions), all of the ``process_utils`` functions do not close `inheritable ` file descriptors before starting subprocesses. +This is equivalent to passing ``close_fds=False`` to ``subprocess.Popen`` on all Python versions. -In Python 3.2, the ``subprocess.Popen`` default for the ``close_fds`` option changed from ``False`` to ``True`` so that file descriptors opened by the parent process were closed before spawning the child process. In Python 3.4, `PEP 0446 `_ additionally made it so even when ``close_fds=False`` file descriptors which are `non-inheritable `_ are still closed before spawning the subprocess. +For historical context, in Python 3.2, the ``subprocess.Popen`` default for the ``close_fds`` option changed from ``False`` to ``True`` so that file descriptors opened by the parent process were closed before spawning the child process. +In Python 3.4, `PEP 0446 `_ additionally made it so even when ``close_fds=False`` file descriptors which are `non-inheritable `_ are still closed before spawning the subprocess. If you want to be able to pass file descriptors to subprocesses in Python 3.4 or higher, you will need to make sure they are `inheritable `. @@ -47,7 +49,7 @@ For synchronous execution and output capture of subprocess, there are two functi - :py:func:`osrf_pycommon.process_utils.execute_process` - :py:func:`osrf_pycommon.process_utils.execute_process_split` -These functions are not yet using the ``trollius``/``asyncio`` framework as a back-end and therefore on Windows will not stream the data from the subprocess as it does on Unix machines. +These functions are not yet using the ``asyncio`` framework as a back-end and therefore on Windows will not stream the data from the subprocess as it does on Unix machines. Instead data will not be yielded until the subprocess is finished and all output is buffered (the normal warnings about long running programs with lots of output apply). The streaming of output does not work on Windows because on Windows the :py:func:`select.select` method only works on sockets and not file-like objects which are used with subprocess pipes. diff --git a/osrf_pycommon/cli_utils/verb_pattern.py b/osrf_pycommon/cli_utils/verb_pattern.py index 8259db9..549364c 100644 --- a/osrf_pycommon/cli_utils/verb_pattern.py +++ b/osrf_pycommon/cli_utils/verb_pattern.py @@ -17,7 +17,10 @@ import sys import inspect -import pkg_resources +try: + import importlib.metadata as importlib_metadata +except ModuleNotFoundError: + import importlib_metadata def call_prepare_arguments(func, parser, sysargs=None): @@ -106,7 +109,7 @@ def create_subparsers(parser, cmd_name, verbs, group, sysargs, title=None): subparser = parser.add_subparsers( title=title or '{0} command'.format(cmd_name), metavar=metavar, - description='Call `{0} {1} -h` for help on a each verb.'.format( + description='Call `{0} {1} -h` for help on each verb.'.format( cmd_name, metavar), dest='verb' ) @@ -149,7 +152,12 @@ def list_verbs(group): :rtype: list of str """ verbs = [] - for entry_point in pkg_resources.iter_entry_points(group=group): + entry_points = importlib_metadata.entry_points() + if hasattr(entry_points, 'select'): + groups = entry_points.select(group=group) + else: + groups = entry_points.get(group, []) + for entry_point in groups: verbs.append(entry_point.name) return verbs @@ -162,7 +170,12 @@ def load_verb_description(verb_name, group): :returns: verb description :rtype: dict """ - for entry_point in pkg_resources.iter_entry_points(group=group): + entry_points = importlib_metadata.entry_points() + if hasattr(entry_points, 'select'): + groups = entry_points.select(group=group) + else: + groups = entry_points.get(group, []) + for entry_point in groups: if entry_point.name == verb_name: return entry_point.load() diff --git a/osrf_pycommon/process_utils/async_execute_process.py b/osrf_pycommon/process_utils/async_execute_process.py index 072c718..07b5da9 100644 --- a/osrf_pycommon/process_utils/async_execute_process.py +++ b/osrf_pycommon/process_utils/async_execute_process.py @@ -16,18 +16,9 @@ import sys -if sys.version_info >= (3, 4) and 'trollius' not in sys.modules: - # If using Python 3.4 or greater, asyncio is always available. - # However, if trollius has already been imported, use that. - from .async_execute_process_asyncio import async_execute_process - from .async_execute_process_asyncio import get_loop - from .async_execute_process_asyncio import asyncio -else: - # If Python is < 3.3 then a SyntaxError will occur with asyncio - # so we will use Trollius on all platforms below Python 3.4. - from .async_execute_process_trollius import async_execute_process - from .async_execute_process_trollius import get_loop - from .async_execute_process_trollius import asyncio +from .async_execute_process_asyncio import async_execute_process +from .async_execute_process_asyncio import get_loop +from .async_execute_process_asyncio import asyncio __all__ = [ 'async_execute_process', @@ -39,9 +30,7 @@ Coroutine to execute a subprocess and yield the output back asynchronously. This function is meant to be used with the Python :py:mod:`asyncio` module, -which is available via pip with Python 3.3 and built-in to Python 3.4. -On Python >= 2.6 you can use the :py:mod:`trollius` module to get the same -functionality, but without using the new ``yield from`` syntax. +which is available in Python 3.5 or greater. Here is an example of how to use this function: @@ -53,40 +42,16 @@ from osrf_pycommon.process_utils import get_loop - @asyncio.coroutine - def setup(): - transport, protocol = yield from async_execute_process( + async def setup(): + transport, protocol = await async_execute_process( AsyncSubprocessProtocol, ['ls', '/usr']) - returncode = yield from protocol.complete + returncode = await protocol.complete return returncode retcode = get_loop().run_until_complete(setup()) get_loop().close() -That same example using :py:mod:`trollius` would look like this: - -.. code-block:: python - - import trollius as asyncio - from osrf_pycommon.process_utils import async_execute_process - from osrf_pycommon.process_utils import AsyncSubprocessProtocol - from osrf_pycommon.process_utils import get_loop - - - @asyncio.coroutine - def setup(): - transport, protocol = yield asyncio.From(async_execute_process( - AsyncSubprocessProtocol, ['ls', '/usr'])) - returncode = yield asyncio.From(protocol.complete) - raise asyncio.Return(returncode) - - retcode = get_loop().run_until_complete(setup()) - get_loop().close() - -This difference is required because in Python < 3.3 the ``yield from`` syntax -is not valid. - -In both examples, the first argument is the default +Tthe first argument is the default :py:class:`AsyncSubprocessProtocol` protocol class, which simply prints output from stdout to stdout and output from stderr to stderr. @@ -96,7 +61,7 @@ def setup(): ``on_process_exited`` functions. See the documentation for the :py:class:`AsyncSubprocessProtocol` class for -more details, but here is an example which uses asyncio from Python 3.4: +more details, but here is an example which uses asyncio from Python 3.5: .. code-block:: python @@ -123,15 +88,14 @@ def on_process_exited(self, returncode): self.fh.close() - @asyncio.coroutine - def log_command_to_file(cmd, file_name): + async def log_command_to_file(cmd, file_name): def create_protocol(**kwargs): return MyProtocol(file_name, **kwargs) - transport, protocol = yield from async_execute_process( + transport, protocol = await async_execute_process( create_protocol, cmd) - returncode = yield from protocol.complete + returncode = await protocol.complete return returncode get_loop().run_until_complete( @@ -179,7 +143,7 @@ def on_process_exited(self, returncode): stdout and stderr and does nothing when the process exits. Data received by the ``on_stdout_received`` and ``on_stderr_received`` - functions is always in bytes (``str`` in Python2 and ``bytes`` in Python3). + functions is always in ``bytes``. Therefore, it may be necessary to call ``.decode()`` on the data before printing to the screen. @@ -225,11 +189,10 @@ def on_stderr_close(self, exc): from osrf_pycommon.process_utils import get_loop - @asyncio.coroutine - def setup(): - transport, protocol = yield from async_execute_process( + async def setup(): + transport, protocol = await async_execute_process( AsyncSubprocessProtocol, ['ls', '-G', '/usr']) - retcode = yield from protocol.complete + retcode = await protocol.complete print("Exited with", retcode) # This will block until the protocol.complete Future is done. diff --git a/osrf_pycommon/process_utils/async_execute_process_trollius.py b/osrf_pycommon/process_utils/async_execute_process_trollius.py deleted file mode 100644 index 2b243a2..0000000 --- a/osrf_pycommon/process_utils/async_execute_process_trollius.py +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright 2014 Open Source Robotics Foundation, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys - -# Conditionally import so that nosetest --with-coverge --cover-inclusive works. -if sys.version_info < (3, 4) or 'trollius' in sys.modules: - import trollius as asyncio - - from trollius import From - from trollius import Return - - try: - import pty - has_pty = True - except ImportError: - has_pty = False - - from .get_loop_impl import get_loop_impl - - def get_loop(): - return get_loop_impl(asyncio) - - @asyncio.coroutine - def _async_execute_process_nopty( - protocol_class, cmd, cwd, env, shell, - stderr_to_stdout=True - ): - loop = get_loop() - stderr = asyncio.subprocess.PIPE - if stderr_to_stdout is True: - stderr = asyncio.subprocess.STDOUT - # Start the subprocess - if shell is True: - transport, protocol = yield From(loop.subprocess_shell( - protocol_class, " ".join(cmd), cwd=cwd, env=env, - stderr=stderr, close_fds=False)) - else: - transport, protocol = yield From(loop.subprocess_exec( - protocol_class, *cmd, cwd=cwd, env=env, - stderr=stderr, close_fds=False)) - raise Return(transport, protocol) - - if has_pty: - # If pty is availabe, use them to emulate the tty - @asyncio.coroutine - def _async_execute_process_pty( - protocol_class, cmd, cwd, env, shell, - stderr_to_stdout=True - ): - loop = get_loop() - # Create the PTY's - stdout_master, stdout_slave = pty.openpty() - if stderr_to_stdout: - stderr_master, stderr_slave = stdout_master, stdout_slave - else: - stderr_master, stderr_slave = pty.openpty() - - def protocol_factory(): - return protocol_class( - stdin=None, - stdout=stdout_master, - stderr=stderr_master - ) - - # Start the subprocess - if shell is True: - transport, protocol = yield From(loop.subprocess_shell( - protocol_factory, " ".join(cmd), cwd=cwd, env=env, - stdout=stdout_slave, stderr=stderr_slave, close_fds=False)) - else: - transport, protocol = yield From(loop.subprocess_exec( - protocol_factory, *cmd, cwd=cwd, env=env, - stdout=stdout_slave, stderr=stderr_slave, close_fds=False)) - - # Close our copies of the slaves, - # the child's copy of the slave remain open until it terminates - os.close(stdout_slave) - if not stderr_to_stdout: - os.close(stderr_slave) - - # Create Protocol classes - class PtyStdoutProtocol(asyncio.Protocol): - def connection_made(self, transport): - if hasattr(protocol, 'on_stdout_open'): - protocol.on_stdout_open() - - def data_received(self, data): - if hasattr(protocol, 'on_stdout_received'): - protocol.on_stdout_received(data) - - def connection_lost(self, exc): - if hasattr(protocol, 'on_stdout_close'): - protocol.on_stdout_close(exc) - - class PtyStderrProtocol(asyncio.Protocol): - def connection_made(self, transport): - if hasattr(protocol, 'on_stderr_open'): - protocol.on_stderr_open() - - def data_received(self, data): - if hasattr(protocol, 'on_stderr_received'): - protocol.on_stderr_received(data) - - def connection_lost(self, exc): - if hasattr(protocol, 'on_stderr_close'): - protocol.on_stderr_close(exc) - - # Add the pty's to the read loop - # Also store the transport, protocol tuple for each call to - # connect_read_pipe, to prevent the destruction of the protocol - # class instance, otherwise no data is received. - protocol.stdout_tuple = yield From(loop.connect_read_pipe( - PtyStdoutProtocol, os.fdopen(stdout_master, 'rb', 0))) - if not stderr_to_stdout: - protocol.stderr_tuple = yield From(loop.connect_read_pipe( - PtyStderrProtocol, os.fdopen(stderr_master, 'rb', 0))) - # Return the protocol and transport - raise Return(transport, protocol) - else: - _async_execute_process_pty = _async_execute_process_nopty - - @asyncio.coroutine - def async_execute_process( - protocol_class, cmd=None, cwd=None, env=None, shell=False, - emulate_tty=False, stderr_to_stdout=True - ): - if emulate_tty: - transport, protocol = yield From(_async_execute_process_pty( - protocol_class, cmd, cwd, env, shell, - stderr_to_stdout)) - else: - transport, protocol = yield From(_async_execute_process_nopty( - protocol_class, cmd, cwd, env, shell, - stderr_to_stdout)) - raise Return(transport, protocol) diff --git a/osrf_pycommon/process_utils/execute_process_nopty.py b/osrf_pycommon/process_utils/execute_process_nopty.py index 57c9a6d..65a7ec1 100644 --- a/osrf_pycommon/process_utils/execute_process_nopty.py +++ b/osrf_pycommon/process_utils/execute_process_nopty.py @@ -30,7 +30,7 @@ def _process_incoming_lines(incoming, left_over): # This function takes the new data, the left over data from last time # and returns a list of complete lines (separated by sep) as well as # any sep trailing data for the next iteration - # This function takes and returns bytes only (str in Python2) + # This function takes and returns bytes only combined = (left_over + incoming) lines = combined.splitlines(True) if not lines: @@ -130,18 +130,14 @@ def yield_to_stream(data, stream): def _execute_process_nopty(cmd, cwd, env, shell, stderr_to_stdout=True): - if stderr_to_stdout: - p = Popen(cmd, - stdin=PIPE, stdout=PIPE, stderr=STDOUT, - cwd=cwd, env=env, shell=shell, close_fds=False) - else: - p = Popen(cmd, - stdin=PIPE, stdout=PIPE, stderr=PIPE, - cwd=cwd, env=env, shell=shell, close_fds=False) - - # Left over data from read which isn't a complete line yet - left_overs = {p.stdout: b'', p.stderr: b''} + stderr = STDOUT if stderr_to_stdout else PIPE + with Popen( + cmd, stdin=PIPE, stdout=PIPE, stderr=stderr, + cwd=cwd, env=env, shell=shell, close_fds=False + ) as p: + # Left over data from read which isn't a complete line yet + left_overs = {p.stdout: b'', p.stderr: b''} - fds = list(filter(None, [p.stdout, p.stderr])) + fds = list(filter(None, [p.stdout, p.stderr])) - return _yield_data(p, fds, left_overs, os.linesep) + yield from _yield_data(p, fds, left_overs, os.linesep) diff --git a/osrf_pycommon/process_utils/get_loop_impl.py b/osrf_pycommon/process_utils/get_loop_impl.py index a9d6349..a798f47 100644 --- a/osrf_pycommon/process_utils/get_loop_impl.py +++ b/osrf_pycommon/process_utils/get_loop_impl.py @@ -14,34 +14,49 @@ import os import threading +import warnings _thread_local = threading.local() def get_loop_impl(asyncio): - global _thread_local - if getattr(_thread_local, 'loop_has_been_setup', False): - return asyncio.get_event_loop() - # Setup this thread's loop and return it - if os.name == 'nt': - try: - loop = asyncio.get_event_loop() - if not isinstance(loop, asyncio.ProactorEventLoop): - # Before replacing the existing loop, explicitly - # close it to prevent an implicit close during - # garbage collection, which may or may not be a - # problem depending on the loop implementation. - loop.close() + # See the note in + # https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.get_event_loop # noqa + # In short, between Python 3.10.0 and 3.10.8, this unconditionally raises a + # DeprecationWarning. But after 3.10.8, it only raises the warning if + # ther eis no current loop set in the policy. Since we are setting a loop + # in the policy, this warning is spurious, and will go away once we get + # away from Python 3.10.6 (verified with Python 3.11.3). + with warnings.catch_warnings(): + warnings.filterwarnings( + 'ignore', + 'There is no current event loop', + DeprecationWarning) + + global _thread_local + if getattr(_thread_local, 'loop_has_been_setup', False): + return asyncio.get_event_loop() + # Setup this thread's loop and return it + if os.name == 'nt': + try: + loop = asyncio.get_event_loop() + if not isinstance(loop, asyncio.ProactorEventLoop): + # Before replacing the existing loop, explicitly + # close it to prevent an implicit close during + # garbage collection, which may or may not be a + # problem depending on the loop implementation. + loop.close() + loop = asyncio.ProactorEventLoop() + asyncio.set_event_loop(loop) + except (RuntimeError, AssertionError): loop = asyncio.ProactorEventLoop() asyncio.set_event_loop(loop) - except (RuntimeError, AssertionError): - loop = asyncio.ProactorEventLoop() - asyncio.set_event_loop(loop) - else: - try: - loop = asyncio.get_event_loop() - except (RuntimeError, AssertionError): - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - _thread_local.loop_has_been_setup = True + else: + try: + loop = asyncio.get_event_loop() + except (RuntimeError, AssertionError): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + _thread_local.loop_has_been_setup = True + return loop diff --git a/osrf_pycommon/process_utils/impl.py b/osrf_pycommon/process_utils/impl.py index 6965079..bff835c 100644 --- a/osrf_pycommon/process_utils/impl.py +++ b/osrf_pycommon/process_utils/impl.py @@ -23,11 +23,6 @@ # so fallback to non pty implementation _execute_process_pty = None -try: - _basestring = basestring # Python 2 -except NameError: - _basestring = str # Python 3 - def execute_process(cmd, cwd=None, env=None, shell=False, emulate_tty=False): """Executes a command with arguments and returns output line by line. diff --git a/package.xml b/package.xml index 5fea581..0c5be13 100644 --- a/package.xml +++ b/package.xml @@ -4,14 +4,16 @@ schematypens="http://www.w3.org/2001/XMLSchema"?> osrf_pycommon - 0.2.1 + 2.1.6 Commonly needed Python modules, used by Python software developed at OSRF. - William Woodall + + William Woodall Apache License 2.0 - python-mock - python3-mock + William Woodall + + python3-importlib-metadata ament_python diff --git a/setup.py b/setup.py index 881f96c..63c8426 100644 --- a/setup.py +++ b/setup.py @@ -1,28 +1,18 @@ -import sys - from setuptools import find_packages from setuptools import setup install_requires = [ - 'setuptools', + 'importlib-metadata;python_version<"3.8"', ] -if sys.version_info < (3, 4): - install_requires.append('trollius') package_excludes = ['tests*', 'docs*'] -if sys.version_info < (3, ): - # On non-Python3 installs, avoid installing the asyncio files - # which contain Python3 specific syntax. - package_excludes.append( - 'osrf_pycommon.process_utils.async_execute_process_asyncio' - ) packages = find_packages(exclude=package_excludes) package_name = 'osrf_pycommon' setup( name=package_name, - version='0.2.1', + version='2.1.6', packages=packages, data_files=[ ('share/' + package_name, ['package.xml']), @@ -30,11 +20,19 @@ ['resource/' + package_name]), ], install_requires=install_requires, + extras_require={ + 'test': [ + 'flake8', + 'flake8_import_order', + 'pytest', + ], + }, + python_requires='>=3.5', zip_safe=True, author='William Woodall', author_email='william@osrfoundation.org', maintainer='William Woodall', - maintainer_email='william@osrfoundation.org', + maintainer_email='william@openrobotics.org', url='http://osrf-pycommon.readthedocs.org/', keywords=['osrf', 'utilities'], classifiers=[ diff --git a/stdeb.cfg b/stdeb.cfg index 16bd367..f13a409 100644 --- a/stdeb.cfg +++ b/stdeb.cfg @@ -1,8 +1,7 @@ [DEFAULT] -Depends: python-setuptools, python-trollius -Depends3: python3-setuptools -Conflicts: python3-osrf-pycommon +Depends3: python3 (>= 3.8) | python3-importlib-metadata Conflicts3: python-osrf-pycommon -Suite: xenial yakkety zesty artful bionic cosmic disco eoan stretch buster -Suite3: xenial yakkety zesty artful bionic cosmic disco eoan focal stretch buster -X-Python3-Version: >= 3.2 +Suite3: focal jammy noble bookworm trixie +X-Python3-Version: >= 3.5 +No-Python2: +Upstream-Version-Suffix: +upstream diff --git a/tests/test_code_format.py b/tests/test_code_format.py index d048c94..2a2e25b 100644 --- a/tests/test_code_format.py +++ b/tests/test_code_format.py @@ -18,9 +18,6 @@ def test_flake8(): cmd.extend(['--ignore=C,D,Q,I']) # work around for https://gitlab.com/pycqa/flake8/issues/179 cmd.extend(['--jobs', '1']) - if sys.version_info < (3, 4): - # Unless Python3, skip files with new syntax, like `yield from` - cmd.append('--exclude={0}/*async_execute_process_asyncio/impl.py'.format(source_dir)) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout, stderr = p.communicate() print(stdout) diff --git a/tests/unit/test_process_utils/impl_aep_asyncio.py b/tests/unit/test_process_utils/impl_aep_asyncio.py index 316a3fb..b3ab108 100644 --- a/tests/unit/test_process_utils/impl_aep_asyncio.py +++ b/tests/unit/test_process_utils/impl_aep_asyncio.py @@ -1,4 +1,3 @@ -from osrf_pycommon.process_utils import asyncio from osrf_pycommon.process_utils.async_execute_process import async_execute_process from osrf_pycommon.process_utils import get_loop @@ -7,9 +6,9 @@ loop = get_loop() -@asyncio.coroutine -def run(cmd, **kwargs): - transport, protocol = yield from async_execute_process( +async def run(cmd, **kwargs): + transport, protocol = await async_execute_process( create_protocol(), cmd, **kwargs) - retcode = yield from protocol.complete + retcode = await protocol.complete + transport.close() return protocol.stdout_buffer, protocol.stderr_buffer, retcode diff --git a/tests/unit/test_process_utils/impl_aep_trollius.py b/tests/unit/test_process_utils/impl_aep_trollius.py deleted file mode 100644 index 7b9cf0e..0000000 --- a/tests/unit/test_process_utils/impl_aep_trollius.py +++ /dev/null @@ -1,25 +0,0 @@ -from osrf_pycommon.process_utils import asyncio -from osrf_pycommon.process_utils.async_execute_process import async_execute_process -from osrf_pycommon.process_utils import get_loop - -# allow module to be importable for --cover-inclusive -try: - from osrf_pycommon.process_utils.async_execute_process_trollius import From -except ImportError: - TROLLIUS_FOUND = False -else: - TROLLIUS_FOUND = True - - from osrf_pycommon.process_utils.async_execute_process_trollius import Return - - from .impl_aep_protocol import create_protocol - - loop = get_loop() - - @asyncio.coroutine - def run(cmd, **kwargs): - transport, protocol = yield From(async_execute_process( - create_protocol(), cmd, **kwargs)) - retcode = yield asyncio.From(protocol.complete) - raise Return(protocol.stdout_buffer, protocol.stderr_buffer, - retcode) diff --git a/tests/unit/test_process_utils/test_async_execute_process.py b/tests/unit/test_process_utils/test_async_execute_process.py index 4c920e0..736f014 100644 --- a/tests/unit/test_process_utils/test_async_execute_process.py +++ b/tests/unit/test_process_utils/test_async_execute_process.py @@ -3,14 +3,8 @@ import sys import unittest -if sys.version_info >= (3, 4): - from .impl_aep_asyncio import run - from .impl_aep_asyncio import loop - print("Using asyncio") -else: - from .impl_aep_trollius import run - from .impl_aep_trollius import loop - print("Using Trollius") +from .impl_aep_asyncio import run +from .impl_aep_asyncio import loop this_dir = os.path.dirname(os.path.abspath(__file__)) diff --git a/tests/unit/test_terminal_utils.py b/tests/unit/test_terminal_utils.py index 6590281..49abfba 100644 --- a/tests/unit/test_terminal_utils.py +++ b/tests/unit/test_terminal_utils.py @@ -1,5 +1,5 @@ -import mock import unittest +from unittest import mock from osrf_pycommon.terminal_utils import is_tty