Merge pull request #335 from goodboy/spawn_backend_table

Spawn backend table

.github/workflows/ci.yml

@@ -26,7 +26,7 @@ jobs:
         run: pip install -U . --upgrade-strategy eager -r requirements-test.txt
 
       - name: Run MyPy check
-        run: mypy tractor/ --ignore-missing-imports
+        run: mypy tractor/ --ignore-missing-imports --show-traceback
 
   # test that we can generate a software distribution and install it
   # thus avoid missing file issues after packaging.
@@ -60,7 +60,11 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         python: ['3.10']
-        spawn_backend: ['trio', 'mp']
+        spawn_backend: [
+          'trio',
+          'mp_spawn',
+          'mp_forkserver',
+        ]
 
     steps:
 

nooz/335.trivial.rst

@@ -0,0 +1,5 @@
+Establish an explicit "backend spawning" method table; use it from CI
+
+More clearly lays out the current set of (3) backends: ``['trio',
+'mp_spawn', 'mp_forkserver']`` and adjusts the ``._spawn.py`` internals
+as well as the test suite to accommodate.

requirements-test.txt

@@ -2,7 +2,7 @@ pytest
 pytest-trio
 pytest-timeout
 pdbpp
-mypy<0.920
-trio_typing<0.7.0
+mypy
+trio_typing
 pexpect
 towncrier

setup.py

@@ -38,7 +38,6 @@
         'tractor',
         'tractor.experimental',
         'tractor.trionics',
-        'tractor.testing',
     ],
     install_requires=[
 

tests/conftest.py

@@ -8,15 +8,89 @@
 import signal
 import platform
 import time
+import inspect
+from functools import partial, wraps
 
 import pytest
+import trio
 import tractor
 
-# export for tests
-from tractor.testing import tractor_test  # noqa
+pytest_plugins = ['pytester']
+
+
+def tractor_test(fn):
+    """
+    Use:
+
+    @tractor_test
+    async def test_whatever():
+        await ...
+
+    If fixtures:
+
+        - ``arb_addr`` (a socket addr tuple where arbiter is listening)
+        - ``loglevel`` (logging level passed to tractor internals)
+        - ``start_method`` (subprocess spawning backend)
+
+    are defined in the `pytest` fixture space they will be automatically
+    injected to tests declaring these funcargs.
+    """
+    @wraps(fn)
+    def wrapper(
+        *args,
+        loglevel=None,
+        arb_addr=None,
+        start_method=None,
+        **kwargs
+    ):
+        # __tracebackhide__ = True
+
+        if 'arb_addr' in inspect.signature(fn).parameters:
+            # injects test suite fixture value to test as well
+            # as `run()`
+            kwargs['arb_addr'] = arb_addr
+
+        if 'loglevel' in inspect.signature(fn).parameters:
+            # allows test suites to define a 'loglevel' fixture
+            # that activates the internal logging
+            kwargs['loglevel'] = loglevel
+
+        if start_method is None:
+            if platform.system() == "Windows":
+                start_method = 'trio'
+
+        if 'start_method' in inspect.signature(fn).parameters:
+            # set of subprocess spawning backends
+            kwargs['start_method'] = start_method
+
+        if kwargs:
+
+            # use explicit root actor start
+
+            async def _main():
+                async with tractor.open_root_actor(
+                    # **kwargs,
+                    arbiter_addr=arb_addr,
+                    loglevel=loglevel,
+                    start_method=start_method,
+
+                    # TODO: only enable when pytest is passed --pdb
+                    # debug_mode=True,
+
+                ):
+                    await fn(*args, **kwargs)
+
+            main = _main
+
+        else:
+            # use implicit root actor start
+            main = partial(fn, *args, **kwargs)
+
+        return trio.run(main)
+
+    return wrapper
 
 
-pytest_plugins = ['pytester']
 _arb_addr = '127.0.0.1', random.randint(1000, 9999)
 
 
@@ -64,11 +138,7 @@ def pytest_addoption(parser):
 
 def pytest_configure(config):
     backend = config.option.spawn_backend
-
-    if backend == 'mp':
-        tractor._spawn.try_set_start_method('spawn')
-    elif backend == 'trio':
-        tractor._spawn.try_set_start_method(backend)
+    tractor._spawn.try_set_start_method(backend)
 
 
 @pytest.fixture(scope='session', autouse=True)
@@ -102,24 +172,24 @@ def arb_addr():
 
 def pytest_generate_tests(metafunc):
     spawn_backend = metafunc.config.option.spawn_backend
+
     if not spawn_backend:
         # XXX some weird windows bug with `pytest`?
-        spawn_backend = 'mp'
-    assert spawn_backend in ('mp', 'trio')
+        spawn_backend = 'trio'
+
+    # TODO: maybe just use the literal `._spawn.SpawnMethodKey`?
+    assert spawn_backend in (
+        'mp_spawn',
+        'mp_forkserver',
+        'trio',
+    )
 
+    # NOTE: used to be used to dyanmically parametrize tests for when
+    # you just passed --spawn-backend=`mp` on the cli, but now we expect
+    # that cli input to be manually specified, BUT, maybe we'll do
+    # something like this again in the future?
     if 'start_method' in metafunc.fixturenames:
-        if spawn_backend == 'mp':
-            from multiprocessing import get_all_start_methods
-            methods = get_all_start_methods()
-            if 'fork' in methods:
-                # fork not available on windows, so check before
-                # removing XXX: the fork method is in general
-                # incompatible with trio's global scheduler state
-                methods.remove('fork')
-        elif spawn_backend == 'trio':
-            methods = ['trio']
-
-        metafunc.parametrize("start_method", methods, scope='module')
+        metafunc.parametrize("start_method", [spawn_backend], scope='module')
 
 
 def sig_prog(proc, sig):

tests/test_cancellation.py

@@ -435,7 +435,7 @@ async def main():
         with trio.fail_after(2):
             async with tractor.open_nursery() as tn:
                 await tn.start_actor('sucka')
-                if spawn_backend == 'mp':
+                if 'mp' in spawn_backend:
                     time.sleep(0.1)
                 os.kill(pid, signal.SIGINT)
                 await trio.sleep_forever()
@@ -474,7 +474,7 @@ async def main():
         with trio.fail_after(timeout):
             async with trio.open_nursery() as n:
                 await n.start(spawn_and_sleep_forever)
-                if spawn_backend == 'mp':
+                if 'mp' in spawn_backend:
                     time.sleep(0.1)
                 os.kill(pid, signal.SIGINT)
 

tests/test_pubsub.py

@@ -4,9 +4,10 @@
 import pytest
 import trio
 import tractor
-from tractor.testing import tractor_test
 from tractor.experimental import msgpub
 
+from conftest import tractor_test
+
 
 def test_type_checks():
 

tests/test_spawning.py

@@ -142,7 +142,7 @@ def test_loglevel_propagated_to_subactor(
     capfd,
     arb_addr,
 ):
-    if start_method == 'forkserver':
+    if start_method == 'mp_forkserver':
         pytest.skip(
             "a bug with `capfd` seems to make forkserver capture not work?")
 

tests/test_streaming.py

@@ -7,9 +7,10 @@
 
 import trio
 import tractor
-from tractor.testing import tractor_test
 import pytest
 
+from conftest import tractor_test
+
 
 def test_must_define_ctx():
 

tractor/_entry.py

@@ -18,26 +18,39 @@
 Sub-process entry points.
 
 """
+from __future__ import annotations
 from functools import partial
-from typing import Any
+from typing import (
+    Any,
+    TYPE_CHECKING,
+)
 
 import trio  # type: ignore
 
-from .log import get_console_log, get_logger
+from .log import (
+    get_console_log,
+    get_logger,
+)
 from . import _state
 from .to_asyncio import run_as_asyncio_guest
-from ._runtime import async_main, Actor
+from ._runtime import (
+    async_main,
+    Actor,
+)
+
+if TYPE_CHECKING:
+    from ._spawn import SpawnMethodKey
 
 
 log = get_logger(__name__)
 
 
 def _mp_main(
 
-    actor: 'Actor',  # type: ignore
+    actor: Actor,  # type: ignore
     accept_addr: tuple[str, int],
     forkserver_info: tuple[Any, Any, Any, Any, Any],
-    start_method: str,
+    start_method: SpawnMethodKey,
     parent_addr: tuple[str, int] = None,
     infect_asyncio: bool = False,
 

tractor/_root.py

@@ -62,7 +62,7 @@ async def open_root_actor(
     # either the `multiprocessing` start method:
     # https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods
     # OR `trio` (the new default).
-    start_method: Optional[str] = None,
+    start_method: Optional[_spawn.SpawnMethodKey] = None,
 
     # enables the multi-process debugger support
     debug_mode: bool = False,