Skip to content

Commit

Permalink
pythongh-76785: Fixes for test.support.interpreters (pythongh-112982)
Browse files Browse the repository at this point in the history
This involves a number of changes for PEP 734.
  • Loading branch information
ericsnowcurrently authored Dec 12, 2023
1 parent f26bfe4 commit 86a77f4
Show file tree
Hide file tree
Showing 30 changed files with 2,506 additions and 1,507 deletions.
15 changes: 14 additions & 1 deletion .github/CODEOWNERS
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,11 @@ Python/traceback.c @iritkatriel
**/*importlib/resources/* @jaraco @warsaw @FFY00
**/importlib/metadata/* @jaraco @warsaw

# Subinterpreters
Lib/test/support/interpreters/** @ericsnowcurrently
Modules/_xx*interp*module.c @ericsnowcurrently
Lib/test/test_interpreters/** @ericsnowcurrently

# Dates and times
**/*datetime* @pganssle @abalkin
**/*str*time* @pganssle @abalkin
Expand Down Expand Up @@ -148,7 +153,15 @@ Doc/c-api/stable.rst @encukou
**/*itertools* @rhettinger
**/*collections* @rhettinger
**/*random* @rhettinger
**/*queue* @rhettinger
Doc/**/*queue* @rhettinger
PCbuild/**/*queue* @rhettinger
Modules/_queuemodule.c @rhettinger
Lib/*queue*.py @rhettinger
Lib/asyncio/*queue*.py @rhettinger
Lib/multiprocessing/*queue*.py @rhettinger
Lib/test/*queue*.py @rhettinger
Lib/test_asyncio/*queue*.py @rhettinger
Lib/test_multiprocessing/*queue*.py @rhettinger
**/*bisect* @rhettinger
**/*heapq* @rhettinger
**/*functools* @rhettinger
Expand Down
10 changes: 10 additions & 0 deletions Include/internal/pycore_crossinterp.h
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,13 @@ extern "C" {
#include "pycore_lock.h" // PyMutex
#include "pycore_pyerrors.h"

/**************/
/* exceptions */
/**************/

PyAPI_DATA(PyObject *) PyExc_InterpreterError;
PyAPI_DATA(PyObject *) PyExc_InterpreterNotFoundError;


/***************************/
/* cross-interpreter calls */
Expand Down Expand Up @@ -160,6 +167,9 @@ struct _xi_state {
extern PyStatus _PyXI_Init(PyInterpreterState *interp);
extern void _PyXI_Fini(PyInterpreterState *interp);

extern PyStatus _PyXI_InitTypes(PyInterpreterState *interp);
extern void _PyXI_FiniTypes(PyInterpreterState *interp);


/***************************/
/* short-term data sharing */
Expand Down
6 changes: 3 additions & 3 deletions Include/internal/pycore_interp.h
Original file line number Diff line number Diff line change
Expand Up @@ -250,9 +250,9 @@ _PyInterpreterState_SetFinalizing(PyInterpreterState *interp, PyThreadState *tst
// Export for the _xxinterpchannels module.
PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpID(int64_t);

extern int _PyInterpreterState_IDInitref(PyInterpreterState *);
extern int _PyInterpreterState_IDIncref(PyInterpreterState *);
extern void _PyInterpreterState_IDDecref(PyInterpreterState *);
PyAPI_FUNC(int) _PyInterpreterState_IDInitref(PyInterpreterState *);
PyAPI_FUNC(int) _PyInterpreterState_IDIncref(PyInterpreterState *);
PyAPI_FUNC(void) _PyInterpreterState_IDDecref(PyInterpreterState *);

extern const PyConfig* _PyInterpreterState_GetConfig(PyInterpreterState *interp);

Expand Down
160 changes: 160 additions & 0 deletions Lib/test/support/interpreters/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
"""Subinterpreters High Level Module."""

import threading
import weakref
import _xxsubinterpreters as _interpreters

# aliases:
from _xxsubinterpreters import (
InterpreterError, InterpreterNotFoundError,
is_shareable,
)


__all__ = [
'get_current', 'get_main', 'create', 'list_all', 'is_shareable',
'Interpreter',
'InterpreterError', 'InterpreterNotFoundError', 'ExecFailure',
'create_queue', 'Queue', 'QueueEmpty', 'QueueFull',
]


_queuemod = None

def __getattr__(name):
if name in ('Queue', 'QueueEmpty', 'QueueFull', 'create_queue'):
global create_queue, Queue, QueueEmpty, QueueFull
ns = globals()
from .queues import (
create as create_queue,
Queue, QueueEmpty, QueueFull,
)
return ns[name]
else:
raise AttributeError(name)


class ExecFailure(RuntimeError):

def __init__(self, excinfo):
msg = excinfo.formatted
if not msg:
if excinfo.type and snapshot.msg:
msg = f'{snapshot.type.__name__}: {snapshot.msg}'
else:
msg = snapshot.type.__name__ or snapshot.msg
super().__init__(msg)
self.snapshot = excinfo


def create():
"""Return a new (idle) Python interpreter."""
id = _interpreters.create(isolated=True)
return Interpreter(id)


def list_all():
"""Return all existing interpreters."""
return [Interpreter(id) for id in _interpreters.list_all()]


def get_current():
"""Return the currently running interpreter."""
id = _interpreters.get_current()
return Interpreter(id)


def get_main():
"""Return the main interpreter."""
id = _interpreters.get_main()
return Interpreter(id)


_known = weakref.WeakValueDictionary()

class Interpreter:
"""A single Python interpreter."""

def __new__(cls, id, /):
# There is only one instance for any given ID.
if not isinstance(id, int):
raise TypeError(f'id must be an int, got {id!r}')
id = int(id)
try:
self = _known[id]
assert hasattr(self, '_ownsref')
except KeyError:
# This may raise InterpreterNotFoundError:
_interpreters._incref(id)
try:
self = super().__new__(cls)
self._id = id
self._ownsref = True
except BaseException:
_interpreters._deccref(id)
raise
_known[id] = self
return self

def __repr__(self):
return f'{type(self).__name__}({self.id})'

def __hash__(self):
return hash(self._id)

def __del__(self):
self._decref()

def _decref(self):
if not self._ownsref:
return
self._ownsref = False
try:
_interpreters._decref(self.id)
except InterpreterNotFoundError:
pass

@property
def id(self):
return self._id

def is_running(self):
"""Return whether or not the identified interpreter is running."""
return _interpreters.is_running(self._id)

def close(self):
"""Finalize and destroy the interpreter.
Attempting to destroy the current interpreter results
in a RuntimeError.
"""
return _interpreters.destroy(self._id)

def exec_sync(self, code, /, channels=None):
"""Run the given source code in the interpreter.
This is essentially the same as calling the builtin "exec"
with this interpreter, using the __dict__ of its __main__
module as both globals and locals.
There is no return value.
If the code raises an unhandled exception then an ExecFailure
is raised, which summarizes the unhandled exception. The actual
exception is discarded because objects cannot be shared between
interpreters.
This blocks the current Python thread until done. During
that time, the previous interpreter is allowed to run
in other threads.
"""
excinfo = _interpreters.exec(self._id, code, channels)
if excinfo is not None:
raise ExecFailure(excinfo)

def run(self, code, /, channels=None):
def task():
self.exec_sync(code, channels=channels)
t = threading.Thread(target=task)
t.start()
return t
Original file line number Diff line number Diff line change
@@ -1,135 +1,23 @@
"""Subinterpreters High Level Module."""
"""Cross-interpreter Channels High Level Module."""

import time
import _xxsubinterpreters as _interpreters
import _xxinterpchannels as _channels

# aliases:
from _xxsubinterpreters import is_shareable
from _xxinterpchannels import (
ChannelError, ChannelNotFoundError, ChannelClosedError,
ChannelEmptyError, ChannelNotEmptyError,
)


__all__ = [
'Interpreter', 'get_current', 'get_main', 'create', 'list_all',
'RunFailedError',
'create', 'list_all',
'SendChannel', 'RecvChannel',
'create_channel', 'list_all_channels', 'is_shareable',
'ChannelError', 'ChannelNotFoundError',
'ChannelEmptyError',
]
'ChannelError', 'ChannelNotFoundError', 'ChannelEmptyError',
]


class RunFailedError(RuntimeError):

def __init__(self, excinfo):
msg = excinfo.formatted
if not msg:
if excinfo.type and snapshot.msg:
msg = f'{snapshot.type.__name__}: {snapshot.msg}'
else:
msg = snapshot.type.__name__ or snapshot.msg
super().__init__(msg)
self.snapshot = excinfo


def create(*, isolated=True):
"""Return a new (idle) Python interpreter."""
id = _interpreters.create(isolated=isolated)
return Interpreter(id, isolated=isolated)


def list_all():
"""Return all existing interpreters."""
return [Interpreter(id) for id in _interpreters.list_all()]


def get_current():
"""Return the currently running interpreter."""
id = _interpreters.get_current()
return Interpreter(id)


def get_main():
"""Return the main interpreter."""
id = _interpreters.get_main()
return Interpreter(id)


class Interpreter:
"""A single Python interpreter."""

def __init__(self, id, *, isolated=None):
if not isinstance(id, (int, _interpreters.InterpreterID)):
raise TypeError(f'id must be an int, got {id!r}')
self._id = id
self._isolated = isolated

def __repr__(self):
data = dict(id=int(self._id), isolated=self._isolated)
kwargs = (f'{k}={v!r}' for k, v in data.items())
return f'{type(self).__name__}({", ".join(kwargs)})'

def __hash__(self):
return hash(self._id)

def __eq__(self, other):
if not isinstance(other, Interpreter):
return NotImplemented
else:
return other._id == self._id

@property
def id(self):
return self._id

@property
def isolated(self):
if self._isolated is None:
# XXX The low-level function has not been added yet.
# See bpo-....
self._isolated = _interpreters.is_isolated(self._id)
return self._isolated

def is_running(self):
"""Return whether or not the identified interpreter is running."""
return _interpreters.is_running(self._id)

def close(self):
"""Finalize and destroy the interpreter.
Attempting to destroy the current interpreter results
in a RuntimeError.
"""
return _interpreters.destroy(self._id)

# XXX Rename "run" to "exec"?
def run(self, src_str, /, channels=None):
"""Run the given source code in the interpreter.
This is essentially the same as calling the builtin "exec"
with this interpreter, using the __dict__ of its __main__
module as both globals and locals.
There is no return value.
If the code raises an unhandled exception then a RunFailedError
is raised, which summarizes the unhandled exception. The actual
exception is discarded because objects cannot be shared between
interpreters.
This blocks the current Python thread until done. During
that time, the previous interpreter is allowed to run
in other threads.
"""
excinfo = _interpreters.exec(self._id, src_str, channels)
if excinfo is not None:
raise RunFailedError(excinfo)


def create_channel():
def create():
"""Return (recv, send) for a new cross-interpreter channel.
The channel may be used to pass data safely between interpreters.
Expand All @@ -139,7 +27,7 @@ def create_channel():
return recv, send


def list_all_channels():
def list_all():
"""Return a list of (recv, send) for all open channels."""
return [(RecvChannel(cid), SendChannel(cid))
for cid in _channels.list_all()]
Expand Down
Loading

0 comments on commit 86a77f4

Please sign in to comment.