diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py index d8e6654fc96efd..8316b5e4a93bb6 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/test/support/interpreters/__init__.py @@ -73,7 +73,7 @@ def __str__(self): def create(): """Return a new (idle) Python interpreter.""" - id = _interpreters.create(isolated=True) + id = _interpreters.create(reqrefs=True) return Interpreter(id) @@ -109,13 +109,13 @@ def __new__(cls, id, /): assert hasattr(self, '_ownsref') except KeyError: # This may raise InterpreterNotFoundError: - _interpreters._incref(id) + _interpreters.incref(id) try: self = super().__new__(cls) self._id = id self._ownsref = True except BaseException: - _interpreters._deccref(id) + _interpreters.decref(id) raise _known[id] = self return self @@ -142,7 +142,7 @@ def _decref(self): return self._ownsref = False try: - _interpreters._decref(self.id) + _interpreters.decref(self.id) except InterpreterNotFoundError: pass diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 35d7355680e549..f674771c27cbb1 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -584,7 +584,7 @@ def f(): def test_create_daemon_thread(self): with self.subTest('isolated'): expected = 'spam spam spam spam spam' - subinterp = interpreters.create(isolated=True) + subinterp = interpreters.create('isolated') script, file = _captured_script(f""" import threading def f(): @@ -604,7 +604,7 @@ def f(): self.assertEqual(out, expected) with self.subTest('not isolated'): - subinterp = interpreters.create(isolated=False) + subinterp = interpreters.create('legacy') script, file = _captured_script(""" import threading def f(): diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 34311afc93fc29..2f2bf03749f834 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2204,6 +2204,7 @@ def test_module_state_shared_in_global(self): self.assertEqual(main_attr_id, subinterp_attr_id) +@requires_subinterpreters class InterpreterConfigTests(unittest.TestCase): supported = { @@ -2277,11 +2278,11 @@ def check(name, expected): expected = self.supported[expected] args = (name,) if name else () - config1 = _testinternalcapi.new_interp_config(*args) + config1 = _interpreters.new_config(*args) self.assert_ns_equal(config1, expected) self.assertIsNot(config1, expected) - config2 = _testinternalcapi.new_interp_config(*args) + config2 = _interpreters.new_config(*args) self.assert_ns_equal(config2, expected) self.assertIsNot(config2, expected) self.assertIsNot(config2, config1) @@ -2298,7 +2299,7 @@ def test_update_from_dict(self): with self.subTest(f'noop ({name})'): expected = vanilla overrides = vars(vanilla) - config = _testinternalcapi.new_interp_config(name, **overrides) + config = _interpreters.new_config(name, **overrides) self.assert_ns_equal(config, expected) with self.subTest(f'change all ({name})'): @@ -2308,7 +2309,7 @@ def test_update_from_dict(self): continue overrides['gil'] = gil expected = types.SimpleNamespace(**overrides) - config = _testinternalcapi.new_interp_config( + config = _interpreters.new_config( name, **overrides) self.assert_ns_equal(config, expected) @@ -2324,14 +2325,14 @@ def test_update_from_dict(self): expected = types.SimpleNamespace( **dict(vars(vanilla), **overrides), ) - config = _testinternalcapi.new_interp_config( + config = _interpreters.new_config( name, **overrides) self.assert_ns_equal(config, expected) with self.subTest('unsupported field'): for name in self.supported: with self.assertRaises(ValueError): - _testinternalcapi.new_interp_config(name, spam=True) + _interpreters.new_config(name, spam=True) # Bad values for bool fields. for field, value in vars(self.supported['empty']).items(): @@ -2341,19 +2342,18 @@ def test_update_from_dict(self): for value in [1, '', 'spam', 1.0, None, object()]: with self.subTest(f'unsupported value ({field}={value!r})'): with self.assertRaises(TypeError): - _testinternalcapi.new_interp_config(**{field: value}) + _interpreters.new_config(**{field: value}) # Bad values for .gil. for value in [True, 1, 1.0, None, object()]: with self.subTest(f'unsupported value(gil={value!r})'): with self.assertRaises(TypeError): - _testinternalcapi.new_interp_config(gil=value) + _interpreters.new_config(gil=value) for value in ['', 'spam']: with self.subTest(f'unsupported value (gil={value!r})'): with self.assertRaises(ValueError): - _testinternalcapi.new_interp_config(gil=value) + _interpreters.new_config(gil=value) - @requires_subinterpreters def test_interp_init(self): questionable = [ # strange @@ -2412,11 +2412,10 @@ def check(config): with self.subTest(f'valid: {config}'): check(config) - @requires_subinterpreters def test_get_config(self): @contextlib.contextmanager def new_interp(config): - interpid = _testinternalcapi.new_interpreter(config) + interpid = _interpreters.create(config, reqrefs=False) try: yield interpid finally: @@ -2426,32 +2425,32 @@ def new_interp(config): pass with self.subTest('main'): - expected = _testinternalcapi.new_interp_config('legacy') + expected = _interpreters.new_config('legacy') expected.gil = 'own' interpid = _interpreters.get_main() - config = _testinternalcapi.get_interp_config(interpid) + config = _interpreters.get_config(interpid) self.assert_ns_equal(config, expected) with self.subTest('isolated'): - expected = _testinternalcapi.new_interp_config('isolated') + expected = _interpreters.new_config('isolated') with new_interp('isolated') as interpid: - config = _testinternalcapi.get_interp_config(interpid) + config = _interpreters.get_config(interpid) self.assert_ns_equal(config, expected) with self.subTest('legacy'): - expected = _testinternalcapi.new_interp_config('legacy') + expected = _interpreters.new_config('legacy') with new_interp('legacy') as interpid: - config = _testinternalcapi.get_interp_config(interpid) + config = _interpreters.get_config(interpid) self.assert_ns_equal(config, expected) with self.subTest('custom'): - orig = _testinternalcapi.new_interp_config( + orig = _interpreters.new_config( 'empty', use_main_obmalloc=True, gil='shared', ) with new_interp(orig) as interpid: - config = _testinternalcapi.get_interp_config(interpid) + config = _interpreters.get_config(interpid) self.assert_ns_equal(config, orig) @@ -2529,14 +2528,19 @@ def test_lookup_destroyed(self): self.assertFalse( _testinternalcapi.interpreter_exists(interpid)) + def get_refcount_helpers(self): + return ( + _testinternalcapi.get_interpreter_refcount, + (lambda id: _interpreters.incref(id, implieslink=False)), + _interpreters.decref, + ) + def test_linked_lifecycle_does_not_exist(self): exists = _testinternalcapi.interpreter_exists is_linked = _testinternalcapi.interpreter_refcount_linked link = _testinternalcapi.link_interpreter_refcount unlink = _testinternalcapi.unlink_interpreter_refcount - get_refcount = _testinternalcapi.get_interpreter_refcount - incref = _testinternalcapi.interpreter_incref - decref = _testinternalcapi.interpreter_decref + get_refcount, incref, decref = self.get_refcount_helpers() with self.subTest('never existed'): interpid = _testinternalcapi.unused_interpreter_id() @@ -2578,8 +2582,7 @@ def test_linked_lifecycle_initial(self): get_refcount = _testinternalcapi.get_interpreter_refcount # A new interpreter will start out not linked, with a refcount of 0. - interpid = _testinternalcapi.new_interpreter() - self.add_interp_cleanup(interpid) + interpid = self.new_interpreter() linked = is_linked(interpid) refcount = get_refcount(interpid) @@ -2589,12 +2592,9 @@ def test_linked_lifecycle_initial(self): def test_linked_lifecycle_never_linked(self): exists = _testinternalcapi.interpreter_exists is_linked = _testinternalcapi.interpreter_refcount_linked - get_refcount = _testinternalcapi.get_interpreter_refcount - incref = _testinternalcapi.interpreter_incref - decref = _testinternalcapi.interpreter_decref + get_refcount, incref, decref = self.get_refcount_helpers() - interpid = _testinternalcapi.new_interpreter() - self.add_interp_cleanup(interpid) + interpid = self.new_interpreter() # Incref will not automatically link it. incref(interpid) @@ -2618,8 +2618,7 @@ def test_linked_lifecycle_link_unlink(self): link = _testinternalcapi.link_interpreter_refcount unlink = _testinternalcapi.unlink_interpreter_refcount - interpid = _testinternalcapi.new_interpreter() - self.add_interp_cleanup(interpid) + interpid = self.new_interpreter() # Linking at refcount 0 does not destroy the interpreter. link(interpid) @@ -2639,12 +2638,9 @@ def test_linked_lifecycle_link_incref_decref(self): exists = _testinternalcapi.interpreter_exists is_linked = _testinternalcapi.interpreter_refcount_linked link = _testinternalcapi.link_interpreter_refcount - get_refcount = _testinternalcapi.get_interpreter_refcount - incref = _testinternalcapi.interpreter_incref - decref = _testinternalcapi.interpreter_decref + get_refcount, incref, decref = self.get_refcount_helpers() - interpid = _testinternalcapi.new_interpreter() - self.add_interp_cleanup(interpid) + interpid = self.new_interpreter() # Linking it will not change the refcount. link(interpid) @@ -2666,11 +2662,9 @@ def test_linked_lifecycle_link_incref_decref(self): def test_linked_lifecycle_incref_link(self): is_linked = _testinternalcapi.interpreter_refcount_linked link = _testinternalcapi.link_interpreter_refcount - get_refcount = _testinternalcapi.get_interpreter_refcount - incref = _testinternalcapi.interpreter_incref + get_refcount, incref, _ = self.get_refcount_helpers() - interpid = _testinternalcapi.new_interpreter() - self.add_interp_cleanup(interpid) + interpid = self.new_interpreter() incref(interpid) self.assertEqual( @@ -2688,12 +2682,9 @@ def test_linked_lifecycle_link_incref_unlink_decref(self): is_linked = _testinternalcapi.interpreter_refcount_linked link = _testinternalcapi.link_interpreter_refcount unlink = _testinternalcapi.unlink_interpreter_refcount - get_refcount = _testinternalcapi.get_interpreter_refcount - incref = _testinternalcapi.interpreter_incref - decref = _testinternalcapi.interpreter_decref + get_refcount, incref, decref = self.get_refcount_helpers() - interpid = _testinternalcapi.new_interpreter() - self.add_interp_cleanup(interpid) + interpid = self.new_interpreter() link(interpid) self.assertTrue( diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 81ec700d9755ce..3c387d973ce0f9 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -2163,7 +2163,7 @@ def re_load(self, name, mod): # subinterpreters def add_subinterpreter(self): - interpid = _interpreters.create(isolated=False) + interpid = _interpreters.create('legacy') def ensure_destroyed(): try: _interpreters.destroy(interpid) diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index a6a76e589761e0..115cb7a56c98f7 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -656,7 +656,7 @@ def test_magic_number(self): class IncompatibleExtensionModuleRestrictionsTests(unittest.TestCase): def run_with_own_gil(self, script): - interpid = _interpreters.create(isolated=True) + interpid = _interpreters.create('isolated') def ensure_destroyed(): try: _interpreters.destroy(interpid) @@ -669,7 +669,7 @@ def ensure_destroyed(): raise ImportError(excsnap.msg) def run_with_shared_gil(self, script): - interpid = _interpreters.create(isolated=False) + interpid = _interpreters.create('legacy') def ensure_destroyed(): try: _interpreters.destroy(interpid) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 3cde9bd0014d9a..2aa7f9bb61aa5b 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -1,13 +1,14 @@ import os import pickle -import threading from textwrap import dedent +import threading +import types import unittest from test import support from test.support import import_helper # Raise SkipTest if subinterpreters not supported. -import_helper.import_module('_xxsubinterpreters') +_interpreters = import_helper.import_module('_xxsubinterpreters') from test.support import interpreters from test.support.interpreters import InterpreterNotFoundError from .utils import _captured_script, _run_output, _running, TestBase @@ -932,6 +933,212 @@ class SubBytes(bytes): interpreters.is_shareable(obj)) +class LowLevelTests(TestBase): + + # The behaviors in the low-level module are important in as much + # as they are exercised by the high-level module. Therefore the + # most important testing happens in the high-level tests. + # These low-level tests cover corner cases that are not + # encountered by the high-level module, thus they + # mostly shouldn't matter as much. + + def test_new_config(self): + # This test overlaps with + # test.test_capi.test_misc.InterpreterConfigTests. + + default = _interpreters.new_config('isolated') + with self.subTest('no arg'): + config = _interpreters.new_config() + self.assert_ns_equal(config, default) + self.assertIsNot(config, default) + + with self.subTest('default'): + config1 = _interpreters.new_config('default') + self.assert_ns_equal(config1, default) + self.assertIsNot(config1, default) + + config2 = _interpreters.new_config('default') + self.assert_ns_equal(config2, config1) + self.assertIsNot(config2, config1) + + for arg in ['', 'default']: + with self.subTest(f'default ({arg!r})'): + config = _interpreters.new_config(arg) + self.assert_ns_equal(config, default) + self.assertIsNot(config, default) + + supported = { + 'isolated': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=True, + allow_daemon_threads=False, + check_multi_interp_extensions=True, + gil='own', + ), + 'legacy': types.SimpleNamespace( + use_main_obmalloc=True, + allow_fork=True, + allow_exec=True, + allow_threads=True, + allow_daemon_threads=True, + check_multi_interp_extensions=False, + gil='shared', + ), + 'empty': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=False, + allow_daemon_threads=False, + check_multi_interp_extensions=False, + gil='default', + ), + } + gil_supported = ['default', 'shared', 'own'] + + for name, vanilla in supported.items(): + with self.subTest(f'supported ({name})'): + expected = vanilla + config1 = _interpreters.new_config(name) + self.assert_ns_equal(config1, expected) + self.assertIsNot(config1, expected) + + config2 = _interpreters.new_config(name) + self.assert_ns_equal(config2, config1) + self.assertIsNot(config2, config1) + + with self.subTest(f'noop override ({name})'): + expected = vanilla + overrides = vars(vanilla) + config = _interpreters.new_config(name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest(f'override all ({name})'): + overrides = {k: not v for k, v in vars(vanilla).items()} + for gil in gil_supported: + if vanilla.gil == gil: + continue + overrides['gil'] = gil + expected = types.SimpleNamespace(**overrides) + config = _interpreters.new_config(name, **overrides) + self.assert_ns_equal(config, expected) + + # Override individual fields. + for field, old in vars(vanilla).items(): + if field == 'gil': + values = [v for v in gil_supported if v != old] + else: + values = [not old] + for val in values: + with self.subTest(f'{name}.{field} ({old!r} -> {val!r})'): + overrides = {field: val} + expected = types.SimpleNamespace( + **dict(vars(vanilla), **overrides), + ) + config = _interpreters.new_config(name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest('extra override'): + with self.assertRaises(ValueError): + _interpreters.new_config(spam=True) + + # Bad values for bool fields. + for field, value in vars(supported['empty']).items(): + if field == 'gil': + continue + assert isinstance(value, bool) + for value in [1, '', 'spam', 1.0, None, object()]: + with self.subTest(f'bad override ({field}={value!r})'): + with self.assertRaises(TypeError): + _interpreters.new_config(**{field: value}) + + # Bad values for .gil. + for value in [True, 1, 1.0, None, object()]: + with self.subTest(f'bad override (gil={value!r})'): + with self.assertRaises(TypeError): + _interpreters.new_config(gil=value) + for value in ['', 'spam']: + with self.subTest(f'bad override (gil={value!r})'): + with self.assertRaises(ValueError): + _interpreters.new_config(gil=value) + + def test_get_config(self): + # This test overlaps with + # test.test_capi.test_misc.InterpreterConfigTests. + + with self.subTest('main'): + expected = _interpreters.new_config('legacy') + expected.gil = 'own' + interpid = _interpreters.get_main() + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('isolated'): + expected = _interpreters.new_config('isolated') + interpid = _interpreters.create('isolated') + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('legacy'): + expected = _interpreters.new_config('legacy') + interpid = _interpreters.create('legacy') + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + def test_create(self): + isolated = _interpreters.new_config('isolated') + legacy = _interpreters.new_config('legacy') + default = isolated + + with self.subTest('no arg'): + interpid = _interpreters.create() + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, default) + + with self.subTest('arg: None'): + interpid = _interpreters.create(None) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, default) + + with self.subTest('arg: \'empty\''): + with self.assertRaises(RuntimeError): + # The "empty" config isn't viable on its own. + _interpreters.create('empty') + + for arg, expected in { + '': default, + 'default': default, + 'isolated': isolated, + 'legacy': legacy, + }.items(): + with self.subTest(f'str arg: {arg!r}'): + interpid = _interpreters.create(arg) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('custom'): + orig = _interpreters.new_config('empty') + orig.use_main_obmalloc = True + orig.gil = 'shared' + interpid = _interpreters.create(orig) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, orig) + + with self.subTest('missing fields'): + orig = _interpreters.new_config() + del orig.gil + with self.assertRaises(ValueError): + _interpreters.create(orig) + + with self.subTest('extra fields'): + orig = _interpreters.new_config() + orig.spam = True + with self.assertRaises(ValueError): + _interpreters.create(orig) + + if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. unittest.main() diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index d16d294b82d044..8ab9ebb354712a 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -28,9 +28,9 @@ def tearDown(self): class LowLevelTests(TestBase): - # The behaviors in the low-level module is important in as much - # as it is exercised by the high-level module. Therefore the - # most # important testing happens in the high-level tests. + # The behaviors in the low-level module are important in as much + # as they are exercised by the high-level module. Therefore the + # most important testing happens in the high-level tests. # These low-level tests cover corner cases that are not # encountered by the high-level module, thus they # mostly shouldn't matter as much. diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index 973d05d4f96dcb..5ade6762ea24ef 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -68,6 +68,9 @@ def run(): class TestBase(unittest.TestCase): + def tearDown(self): + clean_up_interpreters() + def pipe(self): def ensure_closed(fd): try: @@ -156,5 +159,19 @@ def assert_python_failure(self, *argv): self.assertNotEqual(exitcode, 0) return stdout, stderr - def tearDown(self): - clean_up_interpreters() + def assert_ns_equal(self, ns1, ns2, msg=None): + # This is mostly copied from TestCase.assertDictEqual. + self.assertEqual(type(ns1), type(ns2)) + if ns1 == ns2: + return + + import difflib + import pprint + from unittest.util import _common_shorten_repr + standardMsg = '%s != %s' % _common_shorten_repr(ns1, ns2) + diff = ('\n' + '\n'.join(difflib.ndiff( + pprint.pformat(vars(ns1)).splitlines(), + pprint.pformat(vars(ns2)).splitlines()))) + diff = f'namespace({diff})' + standardMsg = self._truncateMessage(standardMsg, diff) + self.fail(self._formatMessage(msg, standardMsg)) diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 56761d1a896d2a..c5d65a373906f2 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -23,7 +23,6 @@ #include "pycore_initconfig.h" // _Py_GetConfigsAsDict() #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() #include "pycore_long.h" // _PyLong_Sign() -#include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_object.h" // _PyObject_IsFreed() #include "pycore_optimizer.h" // _Py_UopsSymbol, etc. #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() @@ -831,6 +830,7 @@ _testinternalcapi_assemble_code_object_impl(PyObject *module, } +// Maybe this could be replaced by get_interpreter_config()? static PyObject * get_interp_settings(PyObject *self, PyObject *args) { @@ -1357,129 +1357,6 @@ dict_getitem_knownhash(PyObject *self, PyObject *args) } -static int -init_named_interp_config(PyInterpreterConfig *config, const char *name) -{ - if (name == NULL) { - name = "isolated"; - } - - if (strcmp(name, "isolated") == 0) { - *config = (PyInterpreterConfig)_PyInterpreterConfig_INIT; - } - else if (strcmp(name, "legacy") == 0) { - *config = (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; - } - else if (strcmp(name, "empty") == 0) { - *config = (PyInterpreterConfig){0}; - } - else { - PyErr_Format(PyExc_ValueError, - "unsupported config name '%s'", name); - return -1; - } - return 0; -} - -static PyObject * -new_interp_config(PyObject *self, PyObject *args, PyObject *kwds) -{ - const char *name = NULL; - if (!PyArg_ParseTuple(args, "|s:new_config", &name)) { - return NULL; - } - PyObject *overrides = kwds; - - if (name == NULL) { - name = "isolated"; - } - - PyInterpreterConfig config; - if (init_named_interp_config(&config, name) < 0) { - return NULL; - } - - if (overrides != NULL && PyDict_GET_SIZE(overrides) > 0) { - if (_PyInterpreterConfig_UpdateFromDict(&config, overrides) < 0) { - return NULL; - } - } - - PyObject *dict = _PyInterpreterConfig_AsDict(&config); - if (dict == NULL) { - return NULL; - } - - PyObject *configobj = _PyNamespace_New(dict); - Py_DECREF(dict); - return configobj; -} - -static PyObject * -get_interp_config(PyObject *self, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"id", NULL}; - PyObject *idobj = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:get_config", kwlist, &idobj)) - { - return NULL; - } - - PyInterpreterState *interp; - if (idobj == NULL) { - interp = PyInterpreterState_Get(); - } - else { - interp = _PyInterpreterState_LookUpIDObject(idobj); - if (interp == NULL) { - return NULL; - } - } - - PyInterpreterConfig config; - if (_PyInterpreterConfig_InitFromState(&config, interp) < 0) { - return NULL; - } - PyObject *dict = _PyInterpreterConfig_AsDict(&config); - if (dict == NULL) { - return NULL; - } - - PyObject *configobj = _PyNamespace_New(dict); - Py_DECREF(dict); - return configobj; -} - -static int -interp_config_from_object(PyObject *configobj, PyInterpreterConfig *config) -{ - if (configobj == NULL || configobj == Py_None) { - if (init_named_interp_config(config, NULL) < 0) { - return -1; - } - } - else if (PyUnicode_Check(configobj)) { - if (init_named_interp_config(config, PyUnicode_AsUTF8(configobj)) < 0) { - return -1; - } - } - else { - PyObject *dict = PyObject_GetAttrString(configobj, "__dict__"); - if (dict == NULL) { - PyErr_Format(PyExc_TypeError, "bad config %R", configobj); - return -1; - } - int res = _PyInterpreterConfig_InitFromDict(config, dict); - Py_DECREF(dict); - if (res < 0) { - return -1; - } - } - return 0; -} - - /* To run some code in a sub-interpreter. */ static PyObject * run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) @@ -1495,7 +1372,14 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) } PyInterpreterConfig config; - if (interp_config_from_object(configobj, &config) < 0) { + PyObject *dict = PyObject_GetAttrString(configobj, "__dict__"); + if (dict == NULL) { + PyErr_Format(PyExc_TypeError, "bad config %R", configobj); + return NULL; + } + int res = _PyInterpreterConfig_InitFromDict(&config, dict); + Py_DECREF(dict); + if (res < 0) { return NULL; } @@ -1546,58 +1430,6 @@ unused_interpreter_id(PyObject *self, PyObject *Py_UNUSED(ignored)) return PyLong_FromLongLong(interpid); } -static PyObject * -new_interpreter(PyObject *self, PyObject *args) -{ - PyObject *configobj = NULL; - if (!PyArg_ParseTuple(args, "|O:new_interpreter", &configobj)) { - return NULL; - } - - PyInterpreterConfig config; - if (interp_config_from_object(configobj, &config) < 0) { - return NULL; - } - - // Unlike _interpreters.create(), we do not automatically link - // the interpreter to its refcount. - PyThreadState *save_tstate = PyThreadState_Get(); - PyThreadState *tstate = NULL; - PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); - PyThreadState_Swap(save_tstate); - if (PyStatus_Exception(status)) { - _PyErr_SetFromPyStatus(status); - return NULL; - } - PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); - - if (_PyInterpreterState_IDInitref(interp) < 0) { - goto error; - } - - int64_t interpid = PyInterpreterState_GetID(interp); - if (interpid < 0) { - goto error; - } - PyObject *idobj = PyLong_FromLongLong(interpid); - if (idobj == NULL) { - goto error; - } - - PyThreadState_Swap(tstate); - PyThreadState_Clear(tstate); - PyThreadState_Swap(save_tstate); - PyThreadState_Delete(tstate); - - return idobj; - -error: - save_tstate = PyThreadState_Swap(tstate); - Py_EndInterpreter(tstate); - PyThreadState_Swap(save_tstate); - return NULL; -} - static PyObject * interpreter_exists(PyObject *self, PyObject *idobj) { @@ -1660,28 +1492,6 @@ interpreter_refcount_linked(PyObject *self, PyObject *idobj) Py_RETURN_FALSE; } -static PyObject * -interpreter_incref(PyObject *self, PyObject *idobj) -{ - PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); - if (interp == NULL) { - return NULL; - } - _PyInterpreterState_IDIncref(interp); - Py_RETURN_NONE; -} - -static PyObject * -interpreter_decref(PyObject *self, PyObject *idobj) -{ - PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); - if (interp == NULL) { - return NULL; - } - _PyInterpreterState_IDDecref(interp); - Py_RETURN_NONE; -} - static void _xid_capsule_destructor(PyObject *capsule) @@ -1928,23 +1738,16 @@ static PyMethodDef module_functions[] = { {"get_object_dict_values", get_object_dict_values, METH_O}, {"hamt", new_hamt, METH_NOARGS}, {"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS}, - {"new_interp_config", _PyCFunction_CAST(new_interp_config), - METH_VARARGS | METH_KEYWORDS}, - {"get_interp_config", _PyCFunction_CAST(get_interp_config), - METH_VARARGS | METH_KEYWORDS}, {"run_in_subinterp_with_config", _PyCFunction_CAST(run_in_subinterp_with_config), METH_VARARGS | METH_KEYWORDS}, {"normalize_interp_id", normalize_interp_id, METH_O}, {"unused_interpreter_id", unused_interpreter_id, METH_NOARGS}, - {"new_interpreter", new_interpreter, METH_VARARGS}, {"interpreter_exists", interpreter_exists, METH_O}, {"get_interpreter_refcount", get_interpreter_refcount, METH_O}, {"link_interpreter_refcount", link_interpreter_refcount, METH_O}, {"unlink_interpreter_refcount", unlink_interpreter_refcount, METH_O}, {"interpreter_refcount_linked", interpreter_refcount_linked, METH_O}, - {"interpreter_incref", interpreter_incref, METH_O}, - {"interpreter_decref", interpreter_decref, METH_O}, {"compile_perf_trampoline_entry", compile_perf_trampoline_entry, METH_VARARGS}, {"perf_trampoline_set_persist_after_fork", perf_trampoline_set_persist_after_fork, METH_VARARGS}, {"get_crossinterp_data", get_crossinterp_data, METH_VARARGS}, diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 5e5b3c10201867..9c2774e4f82def 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -12,8 +12,10 @@ #include "pycore_initconfig.h" // _PyErr_SetFromPyStatus() #include "pycore_long.h" // _PyLong_IsNegative() #include "pycore_modsupport.h" // _PyArg_BadArgument() +#include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_pybuffer.h" // _PyBuffer_ReleaseInInterpreterAndRawFree() #include "pycore_pyerrors.h" // _Py_excinfo +#include "pycore_pylifecycle.h" // _PyInterpreterConfig_AsDict() #include "pycore_pystate.h" // _PyInterpreterState_SetRunningMain() #include "marshal.h" // PyMarshal_ReadObjectFromString() @@ -367,6 +369,115 @@ get_code_str(PyObject *arg, Py_ssize_t *len_p, PyObject **bytes_p, int *flags_p) /* interpreter-specific code ************************************************/ +static int +init_named_config(PyInterpreterConfig *config, const char *name) +{ + if (name == NULL + || strcmp(name, "") == 0 + || strcmp(name, "default") == 0) + { + name = "isolated"; + } + + if (strcmp(name, "isolated") == 0) { + *config = (PyInterpreterConfig)_PyInterpreterConfig_INIT; + } + else if (strcmp(name, "legacy") == 0) { + *config = (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; + } + else if (strcmp(name, "empty") == 0) { + *config = (PyInterpreterConfig){0}; + } + else { + PyErr_Format(PyExc_ValueError, + "unsupported config name '%s'", name); + return -1; + } + return 0; +} + +static int +config_from_object(PyObject *configobj, PyInterpreterConfig *config) +{ + if (configobj == NULL || configobj == Py_None) { + if (init_named_config(config, NULL) < 0) { + return -1; + } + } + else if (PyUnicode_Check(configobj)) { + if (init_named_config(config, PyUnicode_AsUTF8(configobj)) < 0) { + return -1; + } + } + else { + PyObject *dict = PyObject_GetAttrString(configobj, "__dict__"); + if (dict == NULL) { + PyErr_Format(PyExc_TypeError, "bad config %R", configobj); + return -1; + } + int res = _PyInterpreterConfig_InitFromDict(config, dict); + Py_DECREF(dict); + if (res < 0) { + return -1; + } + } + return 0; +} + + +static PyInterpreterState * +new_interpreter(PyInterpreterConfig *config, PyObject **p_idobj, PyThreadState **p_tstate) +{ + PyThreadState *save_tstate = PyThreadState_Get(); + assert(save_tstate != NULL); + PyThreadState *tstate = NULL; + // XXX Possible GILState issues? + PyStatus status = Py_NewInterpreterFromConfig(&tstate, config); + PyThreadState_Swap(save_tstate); + if (PyStatus_Exception(status)) { + /* Since no new thread state was created, there is no exception to + propagate; raise a fresh one after swapping in the old thread + state. */ + _PyErr_SetFromPyStatus(status); + return NULL; + } + assert(tstate != NULL); + PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); + + if (_PyInterpreterState_IDInitref(interp) < 0) { + goto error; + } + + if (p_idobj != NULL) { + // We create the object using the original interpreter. + PyObject *idobj = get_interpid_obj(interp); + if (idobj == NULL) { + goto error; + } + *p_idobj = idobj; + } + + if (p_tstate != NULL) { + *p_tstate = tstate; + } + else { + PyThreadState_Swap(tstate); + PyThreadState_Clear(tstate); + PyThreadState_Swap(save_tstate); + PyThreadState_Delete(tstate); + } + + return interp; + +error: + // XXX Possible GILState issues? + save_tstate = PyThreadState_Swap(tstate); + Py_EndInterpreter(tstate); + PyThreadState_Swap(save_tstate); + return NULL; +} + + static int _run_script(PyObject *ns, const char *codestr, Py_ssize_t codestrlen, int flags) { @@ -436,64 +547,98 @@ _run_in_interpreter(PyInterpreterState *interp, /* module level code ********************************************************/ static PyObject * -interp_create(PyObject *self, PyObject *args, PyObject *kwds) +interp_new_config(PyObject *self, PyObject *args, PyObject *kwds) { + const char *name = NULL; + if (!PyArg_ParseTuple(args, "|s:" MODULE_NAME_STR ".new_config", + &name)) + { + return NULL; + } + PyObject *overrides = kwds; - static char *kwlist[] = {"isolated", NULL}; - int isolated = 1; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|$i:create", kwlist, - &isolated)) { + PyInterpreterConfig config; + if (init_named_config(&config, name) < 0) { return NULL; } - // Create and initialize the new interpreter. - PyThreadState *save_tstate = PyThreadState_Get(); - assert(save_tstate != NULL); - const PyInterpreterConfig config = isolated - ? (PyInterpreterConfig)_PyInterpreterConfig_INIT - : (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; + if (overrides != NULL && PyDict_GET_SIZE(overrides) > 0) { + if (_PyInterpreterConfig_UpdateFromDict(&config, overrides) < 0) { + return NULL; + } + } - // XXX Possible GILState issues? - PyThreadState *tstate = NULL; - PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); - PyThreadState_Swap(save_tstate); - if (PyStatus_Exception(status)) { - /* Since no new thread state was created, there is no exception to - propagate; raise a fresh one after swapping in the old thread - state. */ - _PyErr_SetFromPyStatus(status); + PyObject *dict = _PyInterpreterConfig_AsDict(&config); + if (dict == NULL) { + return NULL; + } + + PyObject *configobj = _PyNamespace_New(dict); + Py_DECREF(dict); + return configobj; +} + +PyDoc_STRVAR(new_config_doc, +"new_config(name='isolated', /, **overrides) -> type.SimpleNamespace\n\ +\n\ +Return a representation of a new PyInterpreterConfig.\n\ +\n\ +The name determines the initial values of the config. Supported named\n\ +configs are: default, isolated, legacy, and empty.\n\ +\n\ +Any keyword arguments are set on the corresponding config fields,\n\ +overriding the initial values."); + + +static PyObject * +interp_create(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"config", "reqrefs", NULL}; + PyObject *configobj = NULL; + int reqrefs = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O$p:create", kwlist, + &configobj, &reqrefs)) { + return NULL; + } + + PyInterpreterConfig config; + if (config_from_object(configobj, &config) < 0) { + return NULL; + } + + PyObject *idobj = NULL; + PyInterpreterState *interp = new_interpreter(&config, &idobj, NULL); + if (interp == NULL) { + // XXX Move the chained exception to interpreters.create()? PyObject *exc = PyErr_GetRaisedException(); + assert(exc != NULL); PyErr_SetString(PyExc_RuntimeError, "interpreter creation failed"); _PyErr_ChainExceptions1(exc); return NULL; } - assert(tstate != NULL); - PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); - PyObject *idobj = get_interpid_obj(interp); - if (idobj == NULL) { - // XXX Possible GILState issues? - save_tstate = PyThreadState_Swap(tstate); - Py_EndInterpreter(tstate); - PyThreadState_Swap(save_tstate); - return NULL; + if (reqrefs) { + // Decref to 0 will destroy the interpreter. + _PyInterpreterState_RequireIDRef(interp, 1); } - PyThreadState_Swap(tstate); - PyThreadState_Clear(tstate); - PyThreadState_Swap(save_tstate); - PyThreadState_Delete(tstate); - - _PyInterpreterState_RequireIDRef(interp, 1); return idobj; } + PyDoc_STRVAR(create_doc, -"create() -> ID\n\ +"create([config], *, reqrefs=False) -> ID\n\ \n\ Create a new interpreter and return a unique generated ID.\n\ \n\ -The caller is responsible for destroying the interpreter before exiting."); +The caller is responsible for destroying the interpreter before exiting,\n\ +typically by using _interpreters.destroy(). This can be managed \n\ +automatically by passing \"reqrefs=True\" and then using _incref() and\n\ +_decref()` appropriately.\n\ +\n\ +\"config\" must be a valid interpreter config or the name of a\n\ +predefined config (\"isolated\" or \"legacy\"). The default\n\ +is \"isolated\"."); static PyObject * @@ -1008,12 +1153,57 @@ Return whether or not the identified interpreter is running."); static PyObject * -interp_incref(PyObject *self, PyObject *args, PyObject *kwds) +interp_get_config(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"id", NULL}; + PyObject *idobj = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O:get_config", kwlist, &idobj)) + { + return NULL; + } + + PyInterpreterState *interp; + if (idobj == NULL) { + interp = PyInterpreterState_Get(); + } + else { + interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + return NULL; + } + } + + PyInterpreterConfig config; + if (_PyInterpreterConfig_InitFromState(&config, interp) < 0) { + return NULL; + } + PyObject *dict = _PyInterpreterConfig_AsDict(&config); + if (dict == NULL) { + return NULL; + } + + PyObject *configobj = _PyNamespace_New(dict); + Py_DECREF(dict); + return configobj; +} + +PyDoc_STRVAR(get_config_doc, +"get_config(id) -> types.SimpleNamespace\n\ +\n\ +Return a representation of the config used to initialize the interpreter."); + + +static PyObject * +interp_incref(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", "implieslink", NULL}; PyObject *id; + int implieslink = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:_incref", kwlist, &id)) { + "O|$p:incref", kwlist, + &id, &implieslink)) + { return NULL; } @@ -1021,8 +1211,10 @@ interp_incref(PyObject *self, PyObject *args, PyObject *kwds) if (interp == NULL) { return NULL; } - if (_PyInterpreterState_IDInitref(interp) < 0) { - return NULL; + + if (implieslink) { + // Decref to 0 will destroy the interpreter. + _PyInterpreterState_RequireIDRef(interp, 1); } _PyInterpreterState_IDIncref(interp); @@ -1036,7 +1228,7 @@ interp_decref(PyObject *self, PyObject *args, PyObject *kwds) static char *kwlist[] = {"id", NULL}; PyObject *id; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:_incref", kwlist, &id)) { + "O:decref", kwlist, &id)) { return NULL; } @@ -1051,6 +1243,8 @@ interp_decref(PyObject *self, PyObject *args, PyObject *kwds) static PyMethodDef module_functions[] = { + {"new_config", _PyCFunction_CAST(interp_new_config), + METH_VARARGS | METH_KEYWORDS, new_config_doc}, {"create", _PyCFunction_CAST(interp_create), METH_VARARGS | METH_KEYWORDS, create_doc}, {"destroy", _PyCFunction_CAST(interp_destroy), @@ -1064,6 +1258,8 @@ static PyMethodDef module_functions[] = { {"is_running", _PyCFunction_CAST(interp_is_running), METH_VARARGS | METH_KEYWORDS, is_running_doc}, + {"get_config", _PyCFunction_CAST(interp_get_config), + METH_VARARGS | METH_KEYWORDS, get_config_doc}, {"exec", _PyCFunction_CAST(interp_exec), METH_VARARGS | METH_KEYWORDS, exec_doc}, {"call", _PyCFunction_CAST(interp_call), @@ -1078,9 +1274,9 @@ static PyMethodDef module_functions[] = { {"is_shareable", _PyCFunction_CAST(object_is_shareable), METH_VARARGS | METH_KEYWORDS, is_shareable_doc}, - {"_incref", _PyCFunction_CAST(interp_incref), + {"incref", _PyCFunction_CAST(interp_incref), METH_VARARGS | METH_KEYWORDS, NULL}, - {"_decref", _PyCFunction_CAST(interp_decref), + {"decref", _PyCFunction_CAST(interp_decref), METH_VARARGS | METH_KEYWORDS, NULL}, {NULL, NULL} /* sentinel */