Cythonize the call loop

.travis.yml

@@ -57,7 +57,7 @@ jobs:
 
 install:
   - pip install -U pip
-  - pip install -U --force-reinstall setuptools tox
+  - pip install -U --force-reinstall setuptools tox cython
 
 script:
   - tox

MANIFEST.in

@@ -5,3 +5,5 @@ include tox.ini
 include LICENSE
 graft testing
 recursive-exclude * *.pyc *.pyo
+include src/pluggy/callers/cythonized.pyx
+include src/pluggy/callers/cythonized.c

pyproject.toml

@@ -3,6 +3,7 @@ requires = [
   "setuptools",
   "setuptools-scm",
   "wheel",
+  "Cython",
 ]
 
 [tool.towncrier]

setup.py

@@ -1,4 +1,7 @@
 from setuptools import setup
+from setuptools.command.sdist import sdist as _sdist
+from setuptools.extension import Extension
+
 
 classifiers = [
     "Development Status :: 4 - Beta",
@@ -29,6 +32,34 @@
 }
 
 
+cmdclass = {}
+
+
+class sdist(_sdist):
+    """Custom sdist building using cython
+    """
+
+    def run(self):
+        # Make sure the compiled Cython files in the distribution
+        # are up-to-date
+        from Cython.Build import cythonize
+
+        cythonize(["src/pluggy/callers/cythonized.pyx"])
+        _sdist.run(self)
+
+
+try:
+    from Cython.Build import cythonize
+
+    print("Building Cython extension(s)")
+    exts = cythonize(["src/pluggy/callers/cythonized.pyx"])
+    cmdclass["sdist"] = sdist
+except ImportError:
+    # When Cython is not installed build from C sources
+    print("Building C extension(s)")
+    exts = [Extension("pluggy.callers.cythonized", ["src/pluggy/callers/cythonized.c"])]
+
+
 def main():
     setup(
         name="pluggy",
@@ -45,8 +76,10 @@ def main():
         install_requires=['importlib-metadata>=0.12;python_version<"3.8"'],
         extras_require=EXTRAS_REQUIRE,
         classifiers=classifiers,
-        packages=["pluggy"],
+        packages=["pluggy", "pluggy.callers"],
         package_dir={"": "src"},
+        ext_modules=exts,
+        cmdclass=cmdclass,
     )
 
 

src/pluggy/callers.py → src/pluggy/callers/__init__.py

@@ -3,7 +3,10 @@
 """
 import sys
 
-from ._result import HookCallError, _Result, _raise_wrapfail
+from .._result import HookCallError, _Result, _raise_wrapfail
+from .cythonized import _c_multicall
+
+__all__ = ["_multicall", "_c_multicall"]
 
 
 def _multicall(hook_impls, caller_kwargs, firstresult=False):

src/pluggy/callers/cythonized.pyx

@@ -0,0 +1,64 @@
+"""
+Cynthonized hook call loop.
+
+This is currently maintained as a verbatim copy of
+``pluggy.callers._multicall()``.
+
+NOTE: In order to build this source you must have cython installed.
+"""
+import sys
+
+from .._result import _Result, _raise_wrapfail, HookCallError
+
+
+cpdef _c_multicall(list hook_impls, dict caller_kwargs, bint firstresult=False):
+    """Execute a call into multiple python functions/methods and return the
+    result(s).
+
+    ``caller_kwargs`` comes from _HookCaller.__call__().
+    """
+    __tracebackhide__ = True
+    results = []
+    excinfo = None
+    try:  # run impl and wrapper setup functions in a loop
+        teardowns = []
+        try:
+            for hook_impl in reversed(hook_impls):
+                try:
+                    args = [caller_kwargs[argname] for argname in hook_impl.argnames]
+                except KeyError:
+                    for argname in hook_impl.argnames:
+                        if argname not in caller_kwargs:
+                            raise HookCallError(
+                                "hook call must provide argument %r" % (argname,))
+
+                if hook_impl.hookwrapper:
+                    try:
+                        gen = hook_impl.function(*args)
+                        next(gen)   # first yield
+                        teardowns.append(gen)
+                    except StopIteration:
+                        _raise_wrapfail(gen, "did not yield")
+                else:
+                    res = hook_impl.function(*args)
+                    if res is not None:
+                        results.append(res)
+                        if firstresult:  # halt further impl calls
+                            break
+        except BaseException:
+            excinfo = sys.exc_info()
+    finally:
+        if firstresult:  # first result hooks return a single value
+            outcome = _Result(results[0] if results else None, excinfo)
+        else:
+            outcome = _Result(results, excinfo)
+
+        # run all wrapper post-yield blocks
+        for gen in reversed(teardowns):
+            try:
+                gen.send(outcome)
+                _raise_wrapfail(gen, "has second yield")
+            except StopIteration:
+                pass
+
+        return outcome.get_result()

testing/benchmark.py

@@ -4,18 +4,18 @@
 import pytest
 from pluggy import HookspecMarker, HookimplMarker
 from pluggy.hooks import HookImpl
-from pluggy.callers import _multicall
+from pluggy.callers import _multicall, _c_multicall
 
 hookspec = HookspecMarker("example")
 hookimpl = HookimplMarker("example")
 
 
-def MC(methods, kwargs, firstresult=False):
+def MC(methods, kwargs, callertype, firstresult=False):
     hookfuncs = []
     for method in methods:
         f = HookImpl(None, "<temp>", method, method.example_impl)
         hookfuncs.append(f)
-    return _multicall(hookfuncs, kwargs, firstresult=firstresult)
+    return callertype(hookfuncs, kwargs, firstresult=firstresult)
 
 
 @hookimpl
@@ -38,9 +38,14 @@ def wrappers(request):
     return [wrapper for i in range(request.param)]
 
 
-def inner_exec(methods):
-    return MC(methods, {"arg1": 1, "arg2": 2, "arg3": 3})
+@pytest.fixture(params=[_multicall, _c_multicall], ids=lambda item: item.__name__)
+def callertype(request):
+    return request.param
 
 
-def test_hook_and_wrappers_speed(benchmark, hooks, wrappers):
-    benchmark(inner_exec, hooks + wrappers)
+def inner_exec(methods, callertype):
+    return MC(methods, {"arg1": 1, "arg2": 2, "arg3": 3}, callertype)
+
+
+def test_hook_and_wrappers_speed(benchmark, hooks, wrappers, callertype):
+    benchmark(inner_exec, hooks + wrappers, callertype)

tox.ini

@@ -19,6 +19,7 @@ commands=pytest {posargs:testing/benchmark.py}
 deps=
   pytest
   pytest-benchmark
+  cython
 
 [testenv:linting]
 skip_install = true