From b022250e67449e0bc49a3c982fe9e6a2d6a7b71a Mon Sep 17 00:00:00 2001 From: Mark Dickinson Date: Sat, 4 Mar 2023 12:20:14 +0000 Subject: [PATCH 01/97] Remove unused internal macros (#102415) Since #101826 was merged, the internal macro `_Py_InIntegralTypeRange` is unused, as are its supporting macros `_Py_IntegralTypeMax` and `_Py_IntegralTypeMin`. This PR removes them. Note that `_Py_InIntegralTypeRange` doesn't actually work as advertised - it's not a safe way to avoid undefined behaviour in an integer to double conversion. --- Include/internal/pycore_pymath.h | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/Include/internal/pycore_pymath.h b/Include/internal/pycore_pymath.h index 5f3afe4df6865c0..7a4e1c1eb714f73 100644 --- a/Include/internal/pycore_pymath.h +++ b/Include/internal/pycore_pymath.h @@ -56,21 +56,6 @@ static inline void _Py_ADJUST_ERANGE2(double x, double y) } } -// Return the maximum value of integral type *type*. -#define _Py_IntegralTypeMax(type) \ - (_Py_IS_TYPE_SIGNED(type) ? (((((type)1 << (sizeof(type)*CHAR_BIT - 2)) - 1) << 1) + 1) : ~(type)0) - -// Return the minimum value of integral type *type*. -#define _Py_IntegralTypeMin(type) \ - (_Py_IS_TYPE_SIGNED(type) ? -_Py_IntegralTypeMax(type) - 1 : 0) - -// Check whether *v* is in the range of integral type *type*. This is most -// useful if *v* is floating-point, since demoting a floating-point *v* to an -// integral type that cannot represent *v*'s integral part is undefined -// behavior. -#define _Py_InIntegralTypeRange(type, v) \ - (_Py_IntegralTypeMin(type) <= v && v <= _Py_IntegralTypeMax(type)) - //--- HAVE_PY_SET_53BIT_PRECISION macro ------------------------------------ // From 705487c6557c3d8866622b4d32528bf7fc2e4204 Mon Sep 17 00:00:00 2001 From: Raj <51259329+workingpayload@users.noreply.github.com> Date: Sat, 4 Mar 2023 19:51:29 +0530 Subject: [PATCH 02/97] gh-101892: Fix `SystemError` when a callable iterator call exhausts the iterator (#101896) Co-authored-by: Oleg Iarygin --- Lib/test/test_iter.py | 25 +++++++++++++++++++ ...-02-14-09-08-48.gh-issue-101892.FMos8l.rst | 3 +++ Objects/iterobject.c | 4 +-- 3 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index 6ab9b7a72303091..30aedb0db3bb3db 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -348,6 +348,31 @@ def spam(state=[0]): return i self.check_iterator(iter(spam, 20), list(range(10)), pickle=False) + def test_iter_function_concealing_reentrant_exhaustion(self): + # gh-101892: Test two-argument iter() with a function that + # exhausts its associated iterator but forgets to either return + # a sentinel value or raise StopIteration. + HAS_MORE = 1 + NO_MORE = 2 + + def exhaust(iterator): + """Exhaust an iterator without raising StopIteration.""" + list(iterator) + + def spam(): + # Touching the iterator with exhaust() below will call + # spam() once again so protect against recursion. + if spam.is_recursive_call: + return NO_MORE + spam.is_recursive_call = True + exhaust(spam.iterator) + return HAS_MORE + + spam.is_recursive_call = False + spam.iterator = iter(spam, NO_MORE) + with self.assertRaises(StopIteration): + next(spam.iterator) + # Test exception propagation through function iterator def test_exception_function(self): def spam(state=[0]): diff --git a/Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst b/Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst new file mode 100644 index 000000000000000..d586779b3a8a365 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst @@ -0,0 +1,3 @@ +Callable iterators no longer raise :class:`SystemError` when the +callable object exhausts the iterator but forgets to either return a +sentinel value or raise :class:`StopIteration`. diff --git a/Objects/iterobject.c b/Objects/iterobject.c index 149b701b68e91a4..7cb17a6ca4ab56a 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -219,7 +219,7 @@ calliter_iternext(calliterobject *it) } result = _PyObject_CallNoArgs(it->it_callable); - if (result != NULL) { + if (result != NULL && it->it_sentinel != NULL){ int ok; ok = PyObject_RichCompareBool(it->it_sentinel, result, Py_EQ); @@ -227,7 +227,6 @@ calliter_iternext(calliterobject *it) return result; /* Common case, fast path */ } - Py_DECREF(result); if (ok > 0) { Py_CLEAR(it->it_callable); Py_CLEAR(it->it_sentinel); @@ -238,6 +237,7 @@ calliter_iternext(calliterobject *it) Py_CLEAR(it->it_callable); Py_CLEAR(it->it_sentinel); } + Py_XDECREF(result); return NULL; } From c2bd55d26f8eb2850eb9f9026b5d7f0ed1420b65 Mon Sep 17 00:00:00 2001 From: Alexey Izbyshev Date: Sat, 4 Mar 2023 17:24:08 +0300 Subject: [PATCH 03/97] gh-102179: Fix `os.dup2` error reporting for negative fds (#102180) --- Lib/test/test_os.py | 20 +++++++++++++++++++ ...-02-23-15-06-01.gh-issue-102179.P6KQ4c.rst | 1 + Modules/posixmodule.c | 5 ----- 3 files changed, 21 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-23-15-06-01.gh-issue-102179.P6KQ4c.rst diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index deea207bfdadd9d..792794ca1094892 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2221,6 +2221,26 @@ def test_closerange(self): def test_dup2(self): self.check(os.dup2, 20) + @unittest.skipUnless(hasattr(os, 'dup2'), 'test needs os.dup2()') + @unittest.skipIf( + support.is_emscripten, + "dup2() with negative fds is broken on Emscripten (see gh-102179)" + ) + def test_dup2_negative_fd(self): + valid_fd = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, valid_fd) + fds = [ + valid_fd, + -1, + -2**31, + ] + for fd, fd2 in itertools.product(fds, repeat=2): + if fd != fd2: + with self.subTest(fd=fd, fd2=fd2): + with self.assertRaises(OSError) as ctx: + os.dup2(fd, fd2) + self.assertEqual(ctx.exception.errno, errno.EBADF) + @unittest.skipUnless(hasattr(os, 'fchmod'), 'test needs os.fchmod()') def test_fchmod(self): self.check(os.fchmod, 0) diff --git a/Misc/NEWS.d/next/Library/2023-02-23-15-06-01.gh-issue-102179.P6KQ4c.rst b/Misc/NEWS.d/next/Library/2023-02-23-15-06-01.gh-issue-102179.P6KQ4c.rst new file mode 100644 index 000000000000000..f77493e267ac7eb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-23-15-06-01.gh-issue-102179.P6KQ4c.rst @@ -0,0 +1 @@ +Fix :func:`os.dup2` error message for negative fds. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 937233a3bd0779e..7beb2cee64a05c6 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -9795,11 +9795,6 @@ os_dup2_impl(PyObject *module, int fd, int fd2, int inheritable) static int dup3_works = -1; #endif - if (fd < 0 || fd2 < 0) { - posix_error(); - return -1; - } - /* dup2() can fail with EINTR if the target FD is already open, because it * then has to be closed. See os_close_impl() for why we don't handle EINTR * upon close(), and therefore below. From 90801e48fdd2a57c5c5a5e202ff76c3a7dc42a50 Mon Sep 17 00:00:00 2001 From: Gouvernathor <44340603+Gouvernathor@users.noreply.github.com> Date: Sat, 4 Mar 2023 16:08:57 +0100 Subject: [PATCH 04/97] gh-102302 Micro-optimize `inspect.Parameter.__hash__` (#102303) --- Lib/inspect.py | 2 +- .../next/Library/2023-03-04-14-46-47.gh-issue-102302.-b_s6Z.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-04-14-46-47.gh-issue-102302.-b_s6Z.rst diff --git a/Lib/inspect.py b/Lib/inspect.py index 8bb3a375735af6a..166667c62cdccf8 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2805,7 +2805,7 @@ def __repr__(self): return '<{} "{}">'.format(self.__class__.__name__, self) def __hash__(self): - return hash((self.name, self.kind, self.annotation, self.default)) + return hash((self._name, self._kind, self._annotation, self._default)) def __eq__(self, other): if self is other: diff --git a/Misc/NEWS.d/next/Library/2023-03-04-14-46-47.gh-issue-102302.-b_s6Z.rst b/Misc/NEWS.d/next/Library/2023-03-04-14-46-47.gh-issue-102302.-b_s6Z.rst new file mode 100644 index 000000000000000..aaf4e62069ca248 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-04-14-46-47.gh-issue-102302.-b_s6Z.rst @@ -0,0 +1 @@ +Micro-optimise hashing of :class:`inspect.Parameter`, reducing the time it takes to hash an instance by around 40%. From 77a3196b7cc17d90a8aae5629aa71ff183b9266a Mon Sep 17 00:00:00 2001 From: Byeongmin Choi Date: Sun, 5 Mar 2023 01:01:54 +0900 Subject: [PATCH 05/97] gh-101863: Fix wrong comments in EUC-KR codec (gh-102417) --- Misc/ACKS | 1 + Modules/cjkcodecs/_codecs_kr.c | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Misc/ACKS b/Misc/ACKS index 2da3d0ab29b81d2..c591cd3bfe4b9e1 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -322,6 +322,7 @@ Adal Chiriliuc Matt Chisholm Lita Cho Kit Yan Choi +Byeongmin Choi Sayan Chowdhury Yuan-Chao Chou Anders Chrigström diff --git a/Modules/cjkcodecs/_codecs_kr.c b/Modules/cjkcodecs/_codecs_kr.c index 6d6acb5c4be4b50..72641e495af0b02 100644 --- a/Modules/cjkcodecs/_codecs_kr.c +++ b/Modules/cjkcodecs/_codecs_kr.c @@ -60,7 +60,7 @@ ENCODER(euc_kr) } else { /* Mapping is found in CP949 extension, - but we encode it in KS X 1001:1998 Annex 3, + but we encode it in KS X 1001:1998, make-up sequence for EUC-KR. */ REQUIRE_OUTBUF(8); @@ -120,7 +120,7 @@ DECODER(euc_kr) if (c == EUCKR_JAMO_FIRSTBYTE && INBYTE2 == EUCKR_JAMO_FILLER) { - /* KS X 1001:1998 Annex 3 make-up sequence */ + /* KS X 1001:1998 make-up sequence */ DBCHAR cho, jung, jong; REQUIRE_INBUF(8); From 81763341ede99ea5ae8993a57b8e3b71b46b2d72 Mon Sep 17 00:00:00 2001 From: Jaysinh Shukla Date: Sun, 5 Mar 2023 03:02:13 +0530 Subject: [PATCH 06/97] gh-63301: Set exit code when tabnanny CLI exits on error (#7699) Co-authored-by: Erlend E. Aasland --- Lib/tabnanny.py | 3 +- Lib/test/test_tabnanny.py | 30 ++++++++++++------- ...3-02-01-10-42-16.gh-issue-63301.XNxSFh.rst | 1 + 3 files changed, 21 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-01-10-42-16.gh-issue-63301.XNxSFh.rst diff --git a/Lib/tabnanny.py b/Lib/tabnanny.py index a47f5a96b897221..9d2df59d36ff477 100755 --- a/Lib/tabnanny.py +++ b/Lib/tabnanny.py @@ -35,6 +35,7 @@ def errprint(*args): sys.stderr.write(sep + str(arg)) sep = " " sys.stderr.write("\n") + sys.exit(1) def main(): import getopt @@ -44,7 +45,6 @@ def main(): opts, args = getopt.getopt(sys.argv[1:], "qv") except getopt.error as msg: errprint(msg) - return for o, a in opts: if o == '-q': filename_only = filename_only + 1 @@ -52,7 +52,6 @@ def main(): verbose = verbose + 1 if not args: errprint("Usage:", sys.argv[0], "[-v] file_or_directory ...") - return for arg in args: check(arg) diff --git a/Lib/test/test_tabnanny.py b/Lib/test/test_tabnanny.py index e0a82e95c486be3..afb8da719b0eeda 100644 --- a/Lib/test/test_tabnanny.py +++ b/Lib/test/test_tabnanny.py @@ -110,9 +110,10 @@ def test_errprint(self): for args, expected in tests: with self.subTest(arguments=args, expected=expected): - with captured_stderr() as stderr: - tabnanny.errprint(*args) - self.assertEqual(stderr.getvalue() , expected) + with self.assertRaises(SystemExit): + with captured_stderr() as stderr: + tabnanny.errprint(*args) + self.assertEqual(stderr.getvalue() , expected) class TestNannyNag(TestCase): @@ -203,14 +204,16 @@ def test_when_wrong_indented(self): err = ('unindent does not match any outer indentation level' ' (, line 3)\n') err = f"{file_path!r}: Indentation Error: {err}" - self.verify_tabnanny_check(file_path, err=err) + with self.assertRaises(SystemExit): + self.verify_tabnanny_check(file_path, err=err) def test_when_tokenize_tokenerror(self): """A python source code file eligible for raising 'tokenize.TokenError'.""" with TemporaryPyFile(SOURCE_CODES["incomplete_expression"]) as file_path: err = "('EOF in multi-line statement', (7, 0))\n" err = f"{file_path!r}: Token Error: {err}" - self.verify_tabnanny_check(file_path, err=err) + with self.assertRaises(SystemExit): + self.verify_tabnanny_check(file_path, err=err) def test_when_nannynag_error_verbose(self): """A python source code file eligible for raising `tabnanny.NannyNag`. @@ -236,7 +239,8 @@ def test_when_no_file(self): path = 'no_file.py' err = (f"{path!r}: I/O Error: [Errno {errno.ENOENT}] " f"{os.strerror(errno.ENOENT)}: {path!r}\n") - self.verify_tabnanny_check(path, err=err) + with self.assertRaises(SystemExit): + self.verify_tabnanny_check(path, err=err) def test_errored_directory(self): """Directory containing wrongly indented python source code files.""" @@ -251,7 +255,8 @@ def test_errored_directory(self): err = ('unindent does not match any outer indentation level' ' (, line 3)\n') err = f"{e_file!r}: Indentation Error: {err}" - self.verify_tabnanny_check(tmp_dir, err=err) + with self.assertRaises(SystemExit): + self.verify_tabnanny_check(tmp_dir, err=err) class TestProcessTokens(TestCase): @@ -287,9 +292,12 @@ def test_with_errored_codes_samples(self): class TestCommandLine(TestCase): """Tests command line interface of `tabnanny`.""" - def validate_cmd(self, *args, stdout="", stderr="", partial=False): + def validate_cmd(self, *args, stdout="", stderr="", partial=False, expect_failure=False): """Common function to assert the behaviour of command line interface.""" - _, out, err = script_helper.assert_python_ok('-m', 'tabnanny', *args) + if expect_failure: + _, out, err = script_helper.assert_python_failure('-m', 'tabnanny', *args) + else: + _, out, err = script_helper.assert_python_ok('-m', 'tabnanny', *args) # Note: The `splitlines()` will solve the problem of CRLF(\r) added # by OS Windows. out = os.fsdecode(out) @@ -310,7 +318,7 @@ def test_with_errored_file(self): stderr = f"{file_path!r}: Indentation Error: " stderr += ('unindent does not match any outer indentation level' ' (, line 3)') - self.validate_cmd(file_path, stderr=stderr) + self.validate_cmd(file_path, stderr=stderr, expect_failure=True) def test_with_error_free_file(self): """Should not display anything if python file is correctly indented.""" @@ -321,7 +329,7 @@ def test_command_usage(self): """Should display usage on no arguments.""" path = findfile('tabnanny.py') stderr = f"Usage: {path} [-v] file_or_directory ..." - self.validate_cmd(stderr=stderr) + self.validate_cmd(stderr=stderr, expect_failure=True) def test_quiet_flag(self): """Should display less when quite mode is on.""" diff --git a/Misc/NEWS.d/next/Library/2023-02-01-10-42-16.gh-issue-63301.XNxSFh.rst b/Misc/NEWS.d/next/Library/2023-02-01-10-42-16.gh-issue-63301.XNxSFh.rst new file mode 100644 index 000000000000000..e00e71fb8554f35 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-01-10-42-16.gh-issue-63301.XNxSFh.rst @@ -0,0 +1 @@ +Set exit code when :mod:`tabnanny` CLI exits on error. From e4609cbe4ca2d3d4fc07c19a7d0bdec52f054c63 Mon Sep 17 00:00:00 2001 From: Dustin Rodrigues Date: Sat, 4 Mar 2023 16:35:25 -0500 Subject: [PATCH 07/97] gh-101992: update pstlib module documentation (#102133) --- Lib/plistlib.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/Lib/plistlib.py b/Lib/plistlib.py index 30f3f673ada5773..3292c30d5fb29b9 100644 --- a/Lib/plistlib.py +++ b/Lib/plistlib.py @@ -21,6 +21,9 @@ Generate Plist example: + import datetime + import plistlib + pl = dict( aString = "Doodah", aList = ["A", "B", 12, 32.1, [1, 2, 3]], @@ -28,22 +31,28 @@ anInt = 728, aDict = dict( anotherString = "", - aUnicodeValue = "M\xe4ssig, Ma\xdf", + aThirdString = "M\xe4ssig, Ma\xdf", aTrueValue = True, aFalseValue = False, ), someData = b"", someMoreData = b"" * 10, - aDate = datetime.datetime.fromtimestamp(time.mktime(time.gmtime())), + aDate = datetime.datetime.now() ) - with open(fileName, 'wb') as fp: - dump(pl, fp) + print(plistlib.dumps(pl).decode()) Parse Plist example: - with open(fileName, 'rb') as fp: - pl = load(fp) - print(pl["aKey"]) + import plistlib + + plist = b''' + + foo + bar + + ''' + pl = plistlib.loads(plist) + print(pl["foo"]) """ __all__ = [ "InvalidFileException", "FMT_XML", "FMT_BINARY", "load", "dump", "loads", "dumps", "UID" From eff9f43924fc836970b2378d58523388d9246194 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20G=C3=B6rgens?= Date: Sun, 5 Mar 2023 05:39:52 +0800 Subject: [PATCH 08/97] gh-96821: Add config option `--with-strict-overflow` (#96823) Co-authored-by: C.A.M. Gerlach Co-authored-by: Erlend E. Aasland Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Co-authored-by: Shantanu --- Doc/using/configure.rst | 5 ++ ...2-09-14-10-38-15.gh-issue-96821.Zk2a9c.rst | 3 + configure | 78 ++++++++++++++++--- configure.ac | 54 ++++++++++--- 4 files changed, 122 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2022-09-14-10-38-15.gh-issue-96821.Zk2a9c.rst diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 8fa8d250d533c93..8936bd381c9d976 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -326,6 +326,11 @@ also be used to improve performance. Enable C-level code profiling with ``gprof`` (disabled by default). +.. cmdoption:: --with-strict-overflow + + Add ``-fstrict-overflow`` to the C compiler flags (by default we add + ``-fno-strict-overflow`` instead). + .. _debug-build: diff --git a/Misc/NEWS.d/next/Build/2022-09-14-10-38-15.gh-issue-96821.Zk2a9c.rst b/Misc/NEWS.d/next/Build/2022-09-14-10-38-15.gh-issue-96821.Zk2a9c.rst new file mode 100644 index 000000000000000..865cfde8b063598 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-09-14-10-38-15.gh-issue-96821.Zk2a9c.rst @@ -0,0 +1,3 @@ +Explicitly mark C extension modules that need defined signed integer overflow, +and add a configure option :option:`--with-strict-overflow`. +Patch by Matthias Görgens and Shantanu Jain. diff --git a/configure b/configure index 557519ad86e06dc..9e99352f589f218 100755 --- a/configure +++ b/configure @@ -1054,6 +1054,7 @@ with_assertions enable_optimizations with_lto enable_bolt +with_strict_overflow with_dsymutil with_address_sanitizer with_memory_sanitizer @@ -1825,6 +1826,8 @@ Optional Packages: --with-lto=[full|thin|no|yes] enable Link-Time-Optimization in any build (default is no) + --with-strict-overflow if 'yes', add -fstrict-overflow to CFLAGS, else add + -fno-strict-overflow (default is no) --with-dsymutil link debug information into final executable with dsymutil in macOS (default is no) --with-address-sanitizer @@ -8376,6 +8379,64 @@ case $CC in fi esac +save_CFLAGS=$CFLAGS +CFLAGS="-fstrict-overflow -fno-strict-overflow" +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking if $CC supports -fstrict-overflow and -fno-strict-overflow" >&5 +$as_echo_n "checking if $CC supports -fstrict-overflow and -fno-strict-overflow... " >&6; } +if ${ac_cv_cc_supports_fstrict_overflow+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ac_cv_cc_supports_fstrict_overflow=yes +else + ac_cv_cc_supports_fstrict_overflow=no + +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_cc_supports_fstrict_overflow" >&5 +$as_echo "$ac_cv_cc_supports_fstrict_overflow" >&6; } +CFLAGS=$save_CFLAGS + +if test "x$ac_cv_cc_supports_fstrict_overflow" = xyes; then : + STRICT_OVERFLOW_CFLAGS="-fstrict-overflow" + NO_STRICT_OVERFLOW_CFLAGS="-fno-strict-overflow" +else + STRICT_OVERFLOW_CFLAGS="" + NO_STRICT_OVERFLOW_CFLAGS="" +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-strict-overflow" >&5 +$as_echo_n "checking for --with-strict-overflow... " >&6; } + +# Check whether --with-strict-overflow was given. +if test "${with_strict_overflow+set}" = set; then : + withval=$with_strict_overflow; + if test "x$ac_cv_cc_supports_fstrict_overflow" = xno; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: --with-strict-overflow=yes requires a compiler that supports -fstrict-overflow" >&5 +$as_echo "$as_me: WARNING: --with-strict-overflow=yes requires a compiler that supports -fstrict-overflow" >&2;} +fi + +else + with_strict_overflow=no + +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_strict_overflow" >&5 +$as_echo "$with_strict_overflow" >&6; } + # Check if CC supports -Og optimization level save_CFLAGS=$CFLAGS CFLAGS="-Og" @@ -8428,15 +8489,8 @@ if test "${OPT-unset}" = "unset" then case $GCC in yes) - # For gcc 4.x we need to use -fwrapv so lets check if its supported - if "$CC" -v --help 2>/dev/null |grep -- -fwrapv > /dev/null; then - WRAP="-fwrapv" - fi - if test -n "${cc_is_clang}" then - # Clang also needs -fwrapv - WRAP="-fwrapv" # bpo-30104: disable strict aliasing to compile correctly dtoa.c, # see Makefile.pre.in for more information CFLAGS_ALIASING="-fno-strict-aliasing" @@ -8447,7 +8501,7 @@ then if test "$Py_DEBUG" = 'true' ; then OPT="-g $PYDEBUG_CFLAGS -Wall" else - OPT="-g $WRAP -O3 -Wall" + OPT="-g -O3 -Wall" fi ;; *) @@ -8580,6 +8634,12 @@ UNIVERSAL_ARCH_FLAGS= # tweak BASECFLAGS based on compiler and platform +if test "x$with_strict_overflow" = xyes; then : + BASECFLAGS="$BASECFLAGS $STRICT_OVERFLOW_CFLAGS" +else + BASECFLAGS="$BASECFLAGS $NO_STRICT_OVERFLOW_CFLAGS" +fi + case $GCC in yes) CFLAGS_NODIST="$CFLAGS_NODIST -std=c11" @@ -27049,7 +27109,7 @@ fi as_fn_append MODULE_BLOCK "MODULE__CTYPES_STATE=$py_cv_module__ctypes$as_nl" if test "x$py_cv_module__ctypes" = xyes; then : - as_fn_append MODULE_BLOCK "MODULE__CTYPES_CFLAGS=$LIBFFI_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__CTYPES_CFLAGS=$NO_STRICT_OVERFLOW_CFLAGS $LIBFFI_CFLAGS$as_nl" as_fn_append MODULE_BLOCK "MODULE__CTYPES_LDFLAGS=$LIBFFI_LIBS$as_nl" fi diff --git a/configure.ac b/configure.ac index 982f669acbcfe5c..31b7a2157a2bccc 100644 --- a/configure.ac +++ b/configure.ac @@ -2073,6 +2073,45 @@ case $CC in fi esac +dnl Historically, some of our code assumed that signed integer overflow +dnl is defined behaviour via twos-complement. +dnl Set STRICT_OVERFLOW_CFLAGS and NO_STRICT_OVERFLOW_CFLAGS depending on compiler support. +dnl Pass the latter to modules that depend on such behaviour. +_SAVE_VAR([CFLAGS]) +CFLAGS="-fstrict-overflow -fno-strict-overflow" +AC_CACHE_CHECK([if $CC supports -fstrict-overflow and -fno-strict-overflow], + [ac_cv_cc_supports_fstrict_overflow], + AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM([[]], [[]])], + [ac_cv_cc_supports_fstrict_overflow=yes], + [ac_cv_cc_supports_fstrict_overflow=no] + ) +) +_RESTORE_VAR([CFLAGS]) + +AS_VAR_IF([ac_cv_cc_supports_fstrict_overflow], [yes], + [STRICT_OVERFLOW_CFLAGS="-fstrict-overflow" + NO_STRICT_OVERFLOW_CFLAGS="-fno-strict-overflow"], + [STRICT_OVERFLOW_CFLAGS="" + NO_STRICT_OVERFLOW_CFLAGS=""]) + +AC_MSG_CHECKING([for --with-strict-overflow]) +AC_ARG_WITH([strict-overflow], + AS_HELP_STRING( + [--with-strict-overflow], + [if 'yes', add -fstrict-overflow to CFLAGS, else add -fno-strict-overflow (default is no)] + ), + [ + AS_VAR_IF( + [ac_cv_cc_supports_fstrict_overflow], [no], + [AC_MSG_WARN([--with-strict-overflow=yes requires a compiler that supports -fstrict-overflow])], + [] + ) + ], + [with_strict_overflow=no] +) +AC_MSG_RESULT([$with_strict_overflow]) + # Check if CC supports -Og optimization level _SAVE_VAR([CFLAGS]) CFLAGS="-Og" @@ -2103,15 +2142,8 @@ if test "${OPT-unset}" = "unset" then case $GCC in yes) - # For gcc 4.x we need to use -fwrapv so lets check if its supported - if "$CC" -v --help 2>/dev/null |grep -- -fwrapv > /dev/null; then - WRAP="-fwrapv" - fi - if test -n "${cc_is_clang}" then - # Clang also needs -fwrapv - WRAP="-fwrapv" # bpo-30104: disable strict aliasing to compile correctly dtoa.c, # see Makefile.pre.in for more information CFLAGS_ALIASING="-fno-strict-aliasing" @@ -2122,7 +2154,7 @@ then if test "$Py_DEBUG" = 'true' ; then OPT="-g $PYDEBUG_CFLAGS -Wall" else - OPT="-g $WRAP -O3 -Wall" + OPT="-g -O3 -Wall" fi ;; *) @@ -2237,6 +2269,10 @@ AC_DEFUN([PY_CHECK_CC_WARNING], [ ]) # tweak BASECFLAGS based on compiler and platform +AS_VAR_IF([with_strict_overflow], [yes], + [BASECFLAGS="$BASECFLAGS $STRICT_OVERFLOW_CFLAGS"], + [BASECFLAGS="$BASECFLAGS $NO_STRICT_OVERFLOW_CFLAGS"]) + case $GCC in yes) CFLAGS_NODIST="$CFLAGS_NODIST -std=c11" @@ -7213,7 +7249,7 @@ PY_STDLIB_MOD([_crypt], [$LIBCRYPT_CFLAGS], [$LIBCRYPT_LIBS]) PY_STDLIB_MOD([_ctypes], [], [test "$have_libffi" = yes], - [$LIBFFI_CFLAGS], [$LIBFFI_LIBS]) + [$NO_STRICT_OVERFLOW_CFLAGS $LIBFFI_CFLAGS], [$LIBFFI_LIBS]) PY_STDLIB_MOD([_curses], [], [test "$have_curses" != "no"], [$CURSES_CFLAGS], [$CURSES_LIBS] From a74cd3ba5de1aad1a1e1ee57328b54c22be47f77 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Sun, 5 Mar 2023 12:15:22 +0530 Subject: [PATCH 09/97] GH-97546: fix flaky asyncio `test_wait_for_race_condition` test (#102421) --- Lib/test/test_asyncio/test_waitfor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_waitfor.py b/Lib/test/test_asyncio/test_waitfor.py index ed80540b2b38520..d5c02ba4a01df99 100644 --- a/Lib/test/test_asyncio/test_waitfor.py +++ b/Lib/test/test_asyncio/test_waitfor.py @@ -159,7 +159,7 @@ async def test_wait_for_race_condition(self): fut = loop.create_future() task = asyncio.wait_for(fut, timeout=0.2) - loop.call_later(0.1, fut.set_result, "ok") + loop.call_soon(fut.set_result, "ok") res = await task self.assertEqual(res, "ok") From 5da379ca7dff44b321450800252be01041b3320b Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Sun, 5 Mar 2023 12:31:56 +0300 Subject: [PATCH 10/97] Move around example in to_bytes() to avoid confusion (#101595) Moves an example to be closer to the sentence that refers to it. --- Doc/library/stdtypes.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 1240f80b0f11f02..550f808a16dfaae 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -530,12 +530,14 @@ class`. In addition, it provides a few more methods: is ``False``. The default values can be used to conveniently turn an integer into a - single byte object. However, when using the default arguments, don't try - to convert a value greater than 255 or you'll get an :exc:`OverflowError`:: + single byte object:: >>> (65).to_bytes() b'A' + However, when using the default arguments, don't try + to convert a value greater than 255 or you'll get an :exc:`OverflowError`. + Equivalent to:: def to_bytes(n, length=1, byteorder='big', signed=False): From 66aa78cbe604a7c5731f074b869f92174a8e3b64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marta=20G=C3=B3mez=20Mac=C3=ADas?= Date: Sun, 5 Mar 2023 12:00:41 +0100 Subject: [PATCH 11/97] gh-102356: Add thrashcan macros to filter object dealloc (#102426) Add thrashcan macros to the deallocator of the filter objects to protect against deeply nested destruction of chains of nested filters. --- Lib/test/test_builtin.py | 10 ++++++++++ Misc/ACKS | 1 + .../2023-03-04-20-56-12.gh-issue-102356.07KvUd.rst | 2 ++ Python/bltinmodule.c | 2 ++ 4 files changed, 15 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-04-20-56-12.gh-issue-102356.07KvUd.rst diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 9e19af0ae90fc1b..e7a79bc13b7f3d9 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -926,6 +926,16 @@ def test_filter_pickle(self): f2 = filter(filter_char, "abcdeabcde") self.check_iter_pickle(f1, list(f2), proto) + def test_filter_dealloc(self): + # Tests recursive deallocation of nested filter objects using the + # thrashcan mechanism. See gh-102356 for more details. + max_iters = 1000000 + i = filter(bool, range(max_iters)) + for _ in range(max_iters): + i = filter(bool, i) + del i + gc.collect() + def test_getattr(self): self.assertTrue(getattr(sys, 'stdout') is sys.stdout) self.assertRaises(TypeError, getattr) diff --git a/Misc/ACKS b/Misc/ACKS index c591cd3bfe4b9e1..7bbde3af99782bb 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -637,6 +637,7 @@ Tim Golden Yonatan Goldschmidt Mark Gollahon Mikhail Golubev +Marta Gómez Macías Guilherme Gonçalves Tiago Gonçalves Chris Gonnerman diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-04-20-56-12.gh-issue-102356.07KvUd.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-04-20-56-12.gh-issue-102356.07KvUd.rst new file mode 100644 index 000000000000000..c03fd5266bc3017 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-04-20-56-12.gh-issue-102356.07KvUd.rst @@ -0,0 +1,2 @@ +Fix a bug that caused a crash when deallocating deeply nested filter +objects. Patch by Marta Gómez Macías. diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 53439ab16040c4f..12ca0ba6c4873cd 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -553,9 +553,11 @@ static void filter_dealloc(filterobject *lz) { PyObject_GC_UnTrack(lz); + Py_TRASHCAN_BEGIN(lz, filter_dealloc) Py_XDECREF(lz->func); Py_XDECREF(lz->it); Py_TYPE(lz)->tp_free(lz); + Py_TRASHCAN_END } static int From 9a478be1a4314734c697dda7a7b0e633a6fb0751 Mon Sep 17 00:00:00 2001 From: Yeojin Kim Date: Sun, 5 Mar 2023 23:54:33 +0900 Subject: [PATCH 12/97] gh-101979: argparse: fix a bug where parentheses in metavar argument of add_argument() were dropped (#102318) --- Lib/argparse.py | 13 ++++++++--- Lib/test/test_argparse.py | 22 +++++++++++++++++++ ...-02-28-09-52-25.gh-issue-101979.or3hXV.rst | 2 ++ 3 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-28-09-52-25.gh-issue-101979.or3hXV.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index 240625ff01084ef..a819d2650e85f03 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -403,10 +403,18 @@ def _format_actions_usage(self, actions, groups): except ValueError: continue else: - end = start + len(group._group_actions) + group_action_count = len(group._group_actions) + end = start + group_action_count if actions[start:end] == group._group_actions: + + suppressed_actions_count = 0 for action in group._group_actions: group_actions.add(action) + if action.help is SUPPRESS: + suppressed_actions_count += 1 + + exposed_actions_count = group_action_count - suppressed_actions_count + if not group.required: if start in inserts: inserts[start] += ' [' @@ -416,7 +424,7 @@ def _format_actions_usage(self, actions, groups): inserts[end] += ']' else: inserts[end] = ']' - else: + elif exposed_actions_count > 1: if start in inserts: inserts[start] += ' (' else: @@ -490,7 +498,6 @@ def _format_actions_usage(self, actions, groups): text = _re.sub(r'(%s) ' % open, r'\1', text) text = _re.sub(r' (%s)' % close, r'\1', text) text = _re.sub(r'%s *%s' % (open, close), r'', text) - text = _re.sub(r'\(([^|]*)\)', r'\1', text) text = text.strip() # return the text diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index cabb2f837693ff4..861da2326d12141 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -3764,6 +3764,28 @@ class TestHelpUsage(HelpTestCase): version = '' +class TestHelpUsageWithParentheses(HelpTestCase): + parser_signature = Sig(prog='PROG') + argument_signatures = [ + Sig('positional', metavar='(example) positional'), + Sig('-p', '--optional', metavar='{1 (option A), 2 (option B)}'), + ] + + usage = '''\ + usage: PROG [-h] [-p {1 (option A), 2 (option B)}] (example) positional + ''' + help = usage + '''\ + + positional arguments: + (example) positional + + options: + -h, --help show this help message and exit + -p {1 (option A), 2 (option B)}, --optional {1 (option A), 2 (option B)} + ''' + version = '' + + class TestHelpOnlyUserGroups(HelpTestCase): """Test basic usage messages""" diff --git a/Misc/NEWS.d/next/Library/2023-02-28-09-52-25.gh-issue-101979.or3hXV.rst b/Misc/NEWS.d/next/Library/2023-02-28-09-52-25.gh-issue-101979.or3hXV.rst new file mode 100644 index 000000000000000..1efe72439b3a4ad --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-28-09-52-25.gh-issue-101979.or3hXV.rst @@ -0,0 +1,2 @@ +Fix a bug where parentheses in the ``metavar`` argument to :meth:`argparse.ArgumentParser.add_argument` were +dropped. Patch by Yeojin Kim. From 7894bbe94ba319eb650f383cb5196424c77b2cfd Mon Sep 17 00:00:00 2001 From: JosephSBoyle <48555120+JosephSBoyle@users.noreply.github.com> Date: Sun, 5 Mar 2023 15:07:44 +0000 Subject: [PATCH 13/97] Fix unused classes in a typing test (GH-102437) As part of investigation issue https://github.com/python/cpython/issues/102433, I discovered what I believe to be an error where two classes `CI` and `DI` are not being used. The assertions beneath them act on `C` and `D`, duplicating existing assertions in this test. Automerge-Triggered-By: GH:AlexWaygood --- Lib/test/test_typing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 96496ec9279e3f9..2eeaf91d78d8f30 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -2921,8 +2921,8 @@ class DI: def __init__(self): self.x = None - self.assertIsInstance(C(), P) - self.assertIsInstance(D(), P) + self.assertIsInstance(CI(), P) + self.assertIsInstance(DI(), P) def test_protocols_in_unions(self): class P(Protocol): From 32220543e2db36c6146ff2704ed1714a6adecc1b Mon Sep 17 00:00:00 2001 From: "Partha P. Mukherjee" Date: Sun, 5 Mar 2023 12:31:26 -0500 Subject: [PATCH 14/97] GH-102341: Improve the test function for pow (#102342) Co-authored-by: Terry Jan Reedy --- Lib/test/test_pow.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_pow.py b/Lib/test/test_pow.py index 5cea9ceb20f5cce..eeb482ec4b27e2d 100644 --- a/Lib/test/test_pow.py +++ b/Lib/test/test_pow.py @@ -19,12 +19,11 @@ def powtest(self, type): self.assertEqual(pow(2, i), pow2) if i != 30 : pow2 = pow2*2 - for othertype in (int,): - for i in list(range(-10, 0)) + list(range(1, 10)): - ii = type(i) - for j in range(1, 11): - jj = -othertype(j) - pow(ii, jj) + for i in list(range(-10, 0)) + list(range(1, 10)): + ii = type(i) + inv = pow(ii, -1) # inverse of ii + for jj in range(-10, 0): + self.assertAlmostEqual(pow(ii, jj), pow(inv, -jj)) for othertype in int, float: for i in range(1, 100): From 96e10229292145012bc462a6ab3ce1626c8acf71 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sun, 5 Mar 2023 21:37:29 +0000 Subject: [PATCH 15/97] gh-102444: Fix minor bugs in `test_typing` highlighted by pyflakes (#102445) --- Lib/test/test_typing.py | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 2eeaf91d78d8f30..0483ca3aa42f94b 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -473,7 +473,6 @@ def test_var_substitution(self): def test_bad_var_substitution(self): T = TypeVar('T') - P = ParamSpec("P") bad_args = ( (), (int, str), Union, Generic, Generic[T], Protocol, Protocol[T], @@ -1037,8 +1036,6 @@ class G2(Generic[Unpack[Ts]]): pass def test_repr_is_correct(self): Ts = TypeVarTuple('Ts') - T = TypeVar('T') - T2 = TypeVar('T2') class G1(Generic[*Ts]): pass class G2(Generic[Unpack[Ts]]): pass @@ -1307,7 +1304,7 @@ def test_callable_args_are_correct(self): i = Callable[[None], *Ts] j = Callable[[None], Unpack[Ts]] self.assertEqual(i.__args__, (type(None), *Ts)) - self.assertEqual(i.__args__, (type(None), Unpack[Ts])) + self.assertEqual(j.__args__, (type(None), Unpack[Ts])) k = Callable[[None], tuple[int, *Ts]] l = Callable[[None], Tuple[int, Unpack[Ts]]] @@ -1435,8 +1432,6 @@ def g(*args: *Ts): pass self.assertEqual(g.__annotations__, {'args': (*Ts,)[0]}) def test_variadic_args_with_ellipsis_annotations_are_correct(self): - Ts = TypeVarTuple('Ts') - def a(*args: *tuple[int, ...]): pass self.assertEqual(a.__annotations__, {'args': (*tuple[int, ...],)[0]}) @@ -4918,7 +4913,6 @@ def test_overload_registry_repeated(self): # Definitions needed for features introduced in Python 3.6 from test import ann_module, ann_module2, ann_module3, ann_module5, ann_module6 -import asyncio T_a = TypeVar('T_a') @@ -7077,16 +7071,6 @@ class C: self.assertEqual(get_type_hints(C, globals())['classvar'], ClassVar[int]) self.assertEqual(get_type_hints(C, globals())['const'], Final[int]) - def test_hash_eq(self): - self.assertEqual(len({Annotated[int, 4, 5], Annotated[int, 4, 5]}), 1) - self.assertNotEqual(Annotated[int, 4, 5], Annotated[int, 5, 4]) - self.assertNotEqual(Annotated[int, 4, 5], Annotated[str, 4, 5]) - self.assertNotEqual(Annotated[int, 4], Annotated[int, 4, 4]) - self.assertEqual( - {Annotated[int, 4, 5], Annotated[int, 4, 5], Annotated[T, 4, 5]}, - {Annotated[int, 4, 5], Annotated[T, 4, 5]} - ) - def test_cannot_subclass(self): with self.assertRaisesRegex(TypeError, "Cannot subclass .*Annotated"): class C(Annotated): @@ -7515,7 +7499,6 @@ class Y(Generic[P, T]): self.assertEqual(B.__args__, ((int, str,), Tuple[bytes, float])) def test_var_substitution(self): - T = TypeVar("T") P = ParamSpec("P") subst = P.__typing_subst__ self.assertEqual(subst((int, str)), (int, str)) @@ -7835,7 +7818,7 @@ def test_special_attrs2(self): self.assertEqual(fr.__module__, 'typing') # Forward refs are currently unpicklable. for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises(TypeError) as exc: + with self.assertRaises(TypeError): pickle.dumps(fr, proto) self.assertEqual(SpecialAttrsTests.TypeName.__name__, 'TypeName') From 3572c861d8e8f03c34793608e7e2221e116aaec0 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sun, 5 Mar 2023 22:00:56 +0000 Subject: [PATCH 16/97] GH-101362: Call join() only when >1 argument supplied to pathlib.PurePath() (#101665) GH-101362: Call join() only when >1 argument supplied to pathlib.PurePath This reduces the time taken to run `PurePath("foo")` by ~15% --- Lib/pathlib.py | 5 ++++- .../Library/2023-02-07-21-16-41.gh-issue-101362.KMQllM.rst | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-07-21-16-41.gh-issue-101362.KMQllM.rst diff --git a/Lib/pathlib.py b/Lib/pathlib.py index dde573592fddced..ed0f2cc73dfa691 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -275,9 +275,12 @@ def __reduce__(self): def _parse_parts(cls, parts): if not parts: return '', '', [] + elif len(parts) == 1: + path = os.fspath(parts[0]) + else: + path = cls._flavour.join(*parts) sep = cls._flavour.sep altsep = cls._flavour.altsep - path = cls._flavour.join(*parts) if altsep: path = path.replace(altsep, sep) drv, root, rel = cls._flavour.splitroot(path) diff --git a/Misc/NEWS.d/next/Library/2023-02-07-21-16-41.gh-issue-101362.KMQllM.rst b/Misc/NEWS.d/next/Library/2023-02-07-21-16-41.gh-issue-101362.KMQllM.rst new file mode 100644 index 000000000000000..af4ee9ad904868e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-07-21-16-41.gh-issue-101362.KMQllM.rst @@ -0,0 +1,2 @@ +Speed up :class:`pathlib.PurePath` construction by calling +:func:`os.path.join` only when two or more arguments are given. From 3e60e0213e9d884a2f4cef4df96956c5d4adde0a Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sun, 5 Mar 2023 22:46:45 +0000 Subject: [PATCH 17/97] GH-101362: Check pathlib.Path flavour compatibility at import time (GH-101664) This saves a comparison in `pathlib.Path.__new__()` and reduces the time taken to run `Path()` by ~5%. Automerge-Triggered-By: GH:AlexWaygood --- Lib/pathlib.py | 16 +++++++++++----- ...023-02-07-20-46-08.gh-issue-101362.2ckZ6R.rst | 2 ++ 2 files changed, 13 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-07-20-46-08.gh-issue-101362.2ckZ6R.rst diff --git a/Lib/pathlib.py b/Lib/pathlib.py index ed0f2cc73dfa691..c37ff21c0352d89 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -707,11 +707,7 @@ def __new__(cls, *args, **kwargs): warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) if cls is Path: cls = WindowsPath if os.name == 'nt' else PosixPath - self = cls._from_parts(args) - if self._flavour is not os.path: - raise NotImplementedError("cannot instantiate %r on your system" - % (cls.__name__,)) - return self + return cls._from_parts(args) def _make_child_relpath(self, part): # This is an optimization used for dir walking. `part` must be @@ -1261,9 +1257,19 @@ class PosixPath(Path, PurePosixPath): """ __slots__ = () + if os.name == 'nt': + def __new__(cls, *args, **kwargs): + raise NotImplementedError( + f"cannot instantiate {cls.__name__!r} on your system") + class WindowsPath(Path, PureWindowsPath): """Path subclass for Windows systems. On a Windows system, instantiating a Path should return this object. """ __slots__ = () + + if os.name != 'nt': + def __new__(cls, *args, **kwargs): + raise NotImplementedError( + f"cannot instantiate {cls.__name__!r} on your system") diff --git a/Misc/NEWS.d/next/Library/2023-02-07-20-46-08.gh-issue-101362.2ckZ6R.rst b/Misc/NEWS.d/next/Library/2023-02-07-20-46-08.gh-issue-101362.2ckZ6R.rst new file mode 100644 index 000000000000000..8421466cdbb3c9f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-07-20-46-08.gh-issue-101362.2ckZ6R.rst @@ -0,0 +1,2 @@ +Speed up :class:`pathlib.Path` construction by running the path flavour +compatibility check only when pathlib is imported. From 6716254e71eeb4666fd6d1a13857832caad7b19f Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sun, 5 Mar 2023 23:50:21 +0000 Subject: [PATCH 18/97] GH-101362: Optimise PurePath(PurePath(...)) (GH-101667) The previous `_parse_args()` method pulled the `_parts` out of any supplied `PurePath` objects; these were subsequently joined in `_from_parts()` using `os.path.join()`. This is actually a slower form of joining than calling `fspath()` on the path object, because it doesn't take advantage of the fact that the contents of `_parts` is normalized! This reduces the time taken to run `PurePath("foo", "bar")` by ~20%, and the time taken to run `PurePath(p, "cheese")`, where `p = PurePath("/foo", "bar", "baz")`, by ~40%. Automerge-Triggered-By: GH:AlexWaygood --- Doc/library/pathlib.rst | 5 +-- Lib/pathlib.py | 36 ++++++------------- Lib/test/test_pathlib.py | 27 ++++++++++++++ ...-02-07-22-20-32.gh-issue-101362.Jlk6mt.rst | 4 +++ 4 files changed, 45 insertions(+), 27 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-07-22-20-32.gh-issue-101362.Jlk6mt.rst diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index c8a734ecad8e7ba..8e91936680fab89 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -105,8 +105,9 @@ we also call *flavours*: PurePosixPath('setup.py') Each element of *pathsegments* can be either a string representing a - path segment, an object implementing the :class:`os.PathLike` interface - which returns a string, or another path object:: + path segment, or an object implementing the :class:`os.PathLike` interface + where the :meth:`~os.PathLike.__fspath__` method returns a string, + such as another path object:: >>> PurePath('foo', 'some/path', 'bar') PurePosixPath('foo/some/path/bar') diff --git a/Lib/pathlib.py b/Lib/pathlib.py index c37ff21c0352d89..d375529ff5f7676 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -281,6 +281,14 @@ def _parse_parts(cls, parts): path = cls._flavour.join(*parts) sep = cls._flavour.sep altsep = cls._flavour.altsep + if isinstance(path, str): + # Force-cast str subclasses to str (issue #21127) + path = str(path) + else: + raise TypeError( + "argument should be a str or an os.PathLike " + "object where __fspath__ returns a str, " + f"not {type(path).__name__!r}") if altsep: path = path.replace(altsep, sep) drv, root, rel = cls._flavour.splitroot(path) @@ -291,32 +299,10 @@ def _parse_parts(cls, parts): parsed = [sys.intern(x) for x in unfiltered_parsed if x and x != '.'] return drv, root, parsed - @classmethod - def _parse_args(cls, args): - # This is useful when you don't want to create an instance, just - # canonicalize some constructor arguments. - parts = [] - for a in args: - if isinstance(a, PurePath): - parts += a._parts - else: - a = os.fspath(a) - if isinstance(a, str): - # Force-cast str subclasses to str (issue #21127) - parts.append(str(a)) - else: - raise TypeError( - "argument should be a str object or an os.PathLike " - "object returning str, not %r" - % type(a)) - return cls._parse_parts(parts) - @classmethod def _from_parts(cls, args): - # We need to call _parse_args on the instance, so as to get the - # right flavour. self = object.__new__(cls) - drv, root, parts = self._parse_args(args) + drv, root, parts = self._parse_parts(args) self._drv = drv self._root = root self._parts = parts @@ -575,7 +561,7 @@ def joinpath(self, *args): anchored). """ drv1, root1, parts1 = self._drv, self._root, self._parts - drv2, root2, parts2 = self._parse_args(args) + drv2, root2, parts2 = self._parse_parts(args) if root2: if not drv2 and drv1: return self._from_parsed_parts(drv1, root2, [drv1 + root2] + parts2[1:]) @@ -662,7 +648,7 @@ def match(self, path_pattern): return True # Can't subclass os.PathLike from PurePath and keep the constructor -# optimizations in PurePath._parse_args(). +# optimizations in PurePath.__slots__. os.PathLike.register(PurePath) diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 4de91d52c6d10c0..df9c1f6ba65deb5 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -166,6 +166,33 @@ def test_constructor_common(self): self.assertEqual(P(P('a'), P('b')), P('a/b')) self.assertEqual(P(P('a'), P('b'), P('c')), P(FakePath("a/b/c"))) + def test_bytes(self): + P = self.cls + message = (r"argument should be a str or an os\.PathLike object " + r"where __fspath__ returns a str, not 'bytes'") + with self.assertRaisesRegex(TypeError, message): + P(b'a') + with self.assertRaises(TypeError): + P(b'a', 'b') + with self.assertRaises(TypeError): + P('a', b'b') + with self.assertRaises(TypeError): + P('a').joinpath(b'b') + with self.assertRaises(TypeError): + P('a') / b'b' + with self.assertRaises(TypeError): + b'a' / P('b') + with self.assertRaises(TypeError): + P('a').match(b'b') + with self.assertRaises(TypeError): + P('a').relative_to(b'b') + with self.assertRaises(TypeError): + P('a').with_name(b'b') + with self.assertRaises(TypeError): + P('a').with_stem(b'b') + with self.assertRaises(TypeError): + P('a').with_suffix(b'b') + def _check_str_subclass(self, *args): # Issue #21127: it should be possible to construct a PurePath object # from a str subclass instance, and it then gets converted to diff --git a/Misc/NEWS.d/next/Library/2023-02-07-22-20-32.gh-issue-101362.Jlk6mt.rst b/Misc/NEWS.d/next/Library/2023-02-07-22-20-32.gh-issue-101362.Jlk6mt.rst new file mode 100644 index 000000000000000..c05f92ae699de96 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-07-22-20-32.gh-issue-101362.Jlk6mt.rst @@ -0,0 +1,4 @@ +Speed up :class:`pathlib.PurePath` construction by handling arguments more +uniformly. When a :class:`pathlib.Path` argument is supplied, +we use its string representation rather than joining its parts +with :func:`os.path.join`. From f533f216e6aaba3f36639ae27210420e7dcf9de1 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Mon, 6 Mar 2023 14:41:53 +0100 Subject: [PATCH 19/97] gh-102416: Do not memoize incorrectly loop rules in the parser (#102467) --- ...-03-06-13-05-33.gh-issue-102416.dz6K5f.rst | 1 + Parser/parser.c | 216 ------------------ Tools/peg_generator/pegen/c_generator.py | 5 +- 3 files changed, 4 insertions(+), 218 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-06-13-05-33.gh-issue-102416.dz6K5f.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-06-13-05-33.gh-issue-102416.dz6K5f.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-06-13-05-33.gh-issue-102416.dz6K5f.rst new file mode 100644 index 000000000000000..9ffc67cfb7ed56a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-06-13-05-33.gh-issue-102416.dz6K5f.rst @@ -0,0 +1 @@ +Do not memoize incorrectly automatically generated loop rules in the parser. Patch by Pablo Galindo. diff --git a/Parser/parser.c b/Parser/parser.c index 845c1739d63a608..e0a88a9cc72c8b8 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -23777,7 +23777,6 @@ _loop0_1_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -23828,7 +23827,6 @@ _loop0_1_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_1_type, _seq); p->level--; return _seq; } @@ -23847,7 +23845,6 @@ _loop0_2_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -23898,7 +23895,6 @@ _loop0_2_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_2_type, _seq); p->level--; return _seq; } @@ -23917,7 +23913,6 @@ _loop1_3_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -23973,7 +23968,6 @@ _loop1_3_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_3_type, _seq); p->level--; return _seq; } @@ -23992,7 +23986,6 @@ _loop0_5_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -24052,7 +24045,6 @@ _loop0_5_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_5_type, _seq); p->level--; return _seq; } @@ -24585,7 +24577,6 @@ _loop1_14_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -24641,7 +24632,6 @@ _loop1_14_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_14_type, _seq); p->level--; return _seq; } @@ -24823,7 +24813,6 @@ _loop0_19_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -24883,7 +24872,6 @@ _loop0_19_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_19_type, _seq); p->level--; return _seq; } @@ -24944,7 +24932,6 @@ _loop0_21_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -25004,7 +24991,6 @@ _loop0_21_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_21_type, _seq); p->level--; return _seq; } @@ -25170,7 +25156,6 @@ _loop0_24_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -25221,7 +25206,6 @@ _loop0_24_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_24_type, _seq); p->level--; return _seq; } @@ -25240,7 +25224,6 @@ _loop1_25_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -25296,7 +25279,6 @@ _loop1_25_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_25_type, _seq); p->level--; return _seq; } @@ -25315,7 +25297,6 @@ _loop0_27_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -25375,7 +25356,6 @@ _loop0_27_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_27_type, _seq); p->level--; return _seq; } @@ -25483,7 +25463,6 @@ _loop0_30_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -25543,7 +25522,6 @@ _loop0_30_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_30_type, _seq); p->level--; return _seq; } @@ -25651,7 +25629,6 @@ _loop1_32_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -25707,7 +25684,6 @@ _loop1_32_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_32_type, _seq); p->level--; return _seq; } @@ -25870,7 +25846,6 @@ _loop0_36_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -25921,7 +25896,6 @@ _loop0_36_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_36_type, _seq); p->level--; return _seq; } @@ -25940,7 +25914,6 @@ _loop0_37_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -25991,7 +25964,6 @@ _loop0_37_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_37_type, _seq); p->level--; return _seq; } @@ -26010,7 +25982,6 @@ _loop0_38_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26061,7 +26032,6 @@ _loop0_38_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_38_type, _seq); p->level--; return _seq; } @@ -26080,7 +26050,6 @@ _loop1_39_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26136,7 +26105,6 @@ _loop1_39_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_39_type, _seq); p->level--; return _seq; } @@ -26155,7 +26123,6 @@ _loop0_40_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26206,7 +26173,6 @@ _loop0_40_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_40_type, _seq); p->level--; return _seq; } @@ -26225,7 +26191,6 @@ _loop1_41_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26281,7 +26246,6 @@ _loop1_41_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_41_type, _seq); p->level--; return _seq; } @@ -26300,7 +26264,6 @@ _loop1_42_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26356,7 +26319,6 @@ _loop1_42_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_42_type, _seq); p->level--; return _seq; } @@ -26375,7 +26337,6 @@ _loop1_43_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26431,7 +26392,6 @@ _loop1_43_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_43_type, _seq); p->level--; return _seq; } @@ -26450,7 +26410,6 @@ _loop0_44_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26501,7 +26460,6 @@ _loop0_44_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_44_type, _seq); p->level--; return _seq; } @@ -26520,7 +26478,6 @@ _loop1_45_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26576,7 +26533,6 @@ _loop1_45_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_45_type, _seq); p->level--; return _seq; } @@ -26595,7 +26551,6 @@ _loop0_46_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26646,7 +26601,6 @@ _loop0_46_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_46_type, _seq); p->level--; return _seq; } @@ -26665,7 +26619,6 @@ _loop1_47_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26721,7 +26674,6 @@ _loop1_47_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_47_type, _seq); p->level--; return _seq; } @@ -26740,7 +26692,6 @@ _loop0_48_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26791,7 +26742,6 @@ _loop0_48_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_48_type, _seq); p->level--; return _seq; } @@ -26810,7 +26760,6 @@ _loop0_49_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26861,7 +26810,6 @@ _loop0_49_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_49_type, _seq); p->level--; return _seq; } @@ -26880,7 +26828,6 @@ _loop1_50_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26936,7 +26883,6 @@ _loop1_50_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_50_type, _seq); p->level--; return _seq; } @@ -26955,7 +26901,6 @@ _loop0_52_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -27015,7 +26960,6 @@ _loop0_52_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_52_type, _seq); p->level--; return _seq; } @@ -27076,7 +27020,6 @@ _loop0_54_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -27136,7 +27079,6 @@ _loop0_54_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_54_type, _seq); p->level--; return _seq; } @@ -27197,7 +27139,6 @@ _loop0_56_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -27257,7 +27198,6 @@ _loop0_56_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_56_type, _seq); p->level--; return _seq; } @@ -27318,7 +27258,6 @@ _loop0_58_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -27378,7 +27317,6 @@ _loop0_58_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_58_type, _seq); p->level--; return _seq; } @@ -27516,7 +27454,6 @@ _loop1_60_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -27572,7 +27509,6 @@ _loop1_60_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_60_type, _seq); p->level--; return _seq; } @@ -27591,7 +27527,6 @@ _loop1_61_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -27647,7 +27582,6 @@ _loop1_61_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_61_type, _seq); p->level--; return _seq; } @@ -27760,7 +27694,6 @@ _loop1_64_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -27816,7 +27749,6 @@ _loop1_64_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_64_type, _seq); p->level--; return _seq; } @@ -27835,7 +27767,6 @@ _loop0_66_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -27895,7 +27826,6 @@ _loop0_66_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_66_type, _seq); p->level--; return _seq; } @@ -28226,7 +28156,6 @@ _loop0_72_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -28286,7 +28215,6 @@ _loop0_72_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_72_type, _seq); p->level--; return _seq; } @@ -28347,7 +28275,6 @@ _loop0_74_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -28407,7 +28334,6 @@ _loop0_74_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_74_type, _seq); p->level--; return _seq; } @@ -28526,7 +28452,6 @@ _loop0_77_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -28586,7 +28511,6 @@ _loop0_77_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_77_type, _seq); p->level--; return _seq; } @@ -28647,7 +28571,6 @@ _loop0_79_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -28707,7 +28630,6 @@ _loop0_79_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_79_type, _seq); p->level--; return _seq; } @@ -28768,7 +28690,6 @@ _loop1_80_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -28824,7 +28745,6 @@ _loop1_80_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_80_type, _seq); p->level--; return _seq; } @@ -28843,7 +28763,6 @@ _loop1_81_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -28899,7 +28818,6 @@ _loop1_81_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_81_type, _seq); p->level--; return _seq; } @@ -28918,7 +28836,6 @@ _loop0_83_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -28978,7 +28895,6 @@ _loop0_83_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_83_type, _seq); p->level--; return _seq; } @@ -29039,7 +28955,6 @@ _loop1_84_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -29095,7 +29010,6 @@ _loop1_84_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_84_type, _seq); p->level--; return _seq; } @@ -29114,7 +29028,6 @@ _loop1_85_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -29170,7 +29083,6 @@ _loop1_85_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_85_type, _seq); p->level--; return _seq; } @@ -29189,7 +29101,6 @@ _loop1_86_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -29245,7 +29156,6 @@ _loop1_86_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_86_type, _seq); p->level--; return _seq; } @@ -29308,7 +29218,6 @@ _loop0_89_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -29368,7 +29277,6 @@ _loop0_89_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_89_type, _seq); p->level--; return _seq; } @@ -29765,7 +29673,6 @@ _loop0_95_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -29816,7 +29723,6 @@ _loop0_95_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_95_type, _seq); p->level--; return _seq; } @@ -29835,7 +29741,6 @@ _loop0_96_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -29886,7 +29791,6 @@ _loop0_96_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_96_type, _seq); p->level--; return _seq; } @@ -29905,7 +29809,6 @@ _loop0_97_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -29956,7 +29859,6 @@ _loop0_97_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_97_type, _seq); p->level--; return _seq; } @@ -29975,7 +29877,6 @@ _loop1_98_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30031,7 +29932,6 @@ _loop1_98_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_98_type, _seq); p->level--; return _seq; } @@ -30050,7 +29950,6 @@ _loop0_99_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30101,7 +30000,6 @@ _loop0_99_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_99_type, _seq); p->level--; return _seq; } @@ -30120,7 +30018,6 @@ _loop1_100_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30176,7 +30073,6 @@ _loop1_100_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_100_type, _seq); p->level--; return _seq; } @@ -30195,7 +30091,6 @@ _loop1_101_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30251,7 +30146,6 @@ _loop1_101_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_101_type, _seq); p->level--; return _seq; } @@ -30270,7 +30164,6 @@ _loop1_102_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30326,7 +30219,6 @@ _loop1_102_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_102_type, _seq); p->level--; return _seq; } @@ -30345,7 +30237,6 @@ _loop0_103_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30396,7 +30287,6 @@ _loop0_103_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_103_type, _seq); p->level--; return _seq; } @@ -30415,7 +30305,6 @@ _loop1_104_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30471,7 +30360,6 @@ _loop1_104_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_104_type, _seq); p->level--; return _seq; } @@ -30490,7 +30378,6 @@ _loop0_105_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30541,7 +30428,6 @@ _loop0_105_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_105_type, _seq); p->level--; return _seq; } @@ -30560,7 +30446,6 @@ _loop1_106_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30616,7 +30501,6 @@ _loop1_106_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_106_type, _seq); p->level--; return _seq; } @@ -30635,7 +30519,6 @@ _loop0_107_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30686,7 +30569,6 @@ _loop0_107_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_107_type, _seq); p->level--; return _seq; } @@ -30705,7 +30587,6 @@ _loop1_108_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30761,7 +30642,6 @@ _loop1_108_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_108_type, _seq); p->level--; return _seq; } @@ -30780,7 +30660,6 @@ _loop1_109_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30836,7 +30715,6 @@ _loop1_109_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_109_type, _seq); p->level--; return _seq; } @@ -30905,7 +30783,6 @@ _loop0_112_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30965,7 +30842,6 @@ _loop0_112_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_112_type, _seq); p->level--; return _seq; } @@ -31026,7 +30902,6 @@ _loop1_113_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -31082,7 +30957,6 @@ _loop1_113_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_113_type, _seq); p->level--; return _seq; } @@ -31101,7 +30975,6 @@ _loop0_114_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -31152,7 +31025,6 @@ _loop0_114_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_114_type, _seq); p->level--; return _seq; } @@ -31171,7 +31043,6 @@ _loop0_115_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -31222,7 +31093,6 @@ _loop0_115_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_115_type, _seq); p->level--; return _seq; } @@ -31301,7 +31171,6 @@ _loop0_118_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -31361,7 +31230,6 @@ _loop0_118_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_118_type, _seq); p->level--; return _seq; } @@ -31470,7 +31338,6 @@ _loop0_121_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -31530,7 +31397,6 @@ _loop0_121_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_121_type, _seq); p->level--; return _seq; } @@ -31591,7 +31457,6 @@ _loop0_123_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -31651,7 +31516,6 @@ _loop0_123_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_123_type, _seq); p->level--; return _seq; } @@ -31712,7 +31576,6 @@ _loop0_125_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -31772,7 +31635,6 @@ _loop0_125_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_125_type, _seq); p->level--; return _seq; } @@ -31833,7 +31695,6 @@ _loop0_127_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -31893,7 +31754,6 @@ _loop0_127_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_127_type, _seq); p->level--; return _seq; } @@ -31954,7 +31814,6 @@ _loop0_128_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -32005,7 +31864,6 @@ _loop0_128_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_128_type, _seq); p->level--; return _seq; } @@ -32024,7 +31882,6 @@ _loop0_130_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -32084,7 +31941,6 @@ _loop0_130_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_130_type, _seq); p->level--; return _seq; } @@ -32145,7 +32001,6 @@ _loop1_131_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -32201,7 +32056,6 @@ _loop1_131_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_131_type, _seq); p->level--; return _seq; } @@ -32261,7 +32115,6 @@ _loop0_134_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -32321,7 +32174,6 @@ _loop0_134_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_134_type, _seq); p->level--; return _seq; } @@ -32382,7 +32234,6 @@ _loop0_136_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -32442,7 +32293,6 @@ _loop0_136_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_136_type, _seq); p->level--; return _seq; } @@ -32503,7 +32353,6 @@ _loop0_138_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -32563,7 +32412,6 @@ _loop0_138_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_138_type, _seq); p->level--; return _seq; } @@ -32624,7 +32472,6 @@ _loop0_140_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -32684,7 +32531,6 @@ _loop0_140_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_140_type, _seq); p->level--; return _seq; } @@ -32745,7 +32591,6 @@ _loop0_142_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -32805,7 +32650,6 @@ _loop0_142_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_142_type, _seq); p->level--; return _seq; } @@ -33557,7 +33401,6 @@ _loop0_154_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -33608,7 +33451,6 @@ _loop0_154_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_154_type, _seq); p->level--; return _seq; } @@ -33627,7 +33469,6 @@ _loop0_155_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -33678,7 +33519,6 @@ _loop0_155_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_155_type, _seq); p->level--; return _seq; } @@ -33697,7 +33537,6 @@ _loop0_156_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -33748,7 +33587,6 @@ _loop0_156_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_156_type, _seq); p->level--; return _seq; } @@ -34076,7 +33914,6 @@ _loop0_162_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -34127,7 +33964,6 @@ _loop0_162_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_162_type, _seq); p->level--; return _seq; } @@ -34146,7 +33982,6 @@ _loop0_163_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -34197,7 +34032,6 @@ _loop0_163_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_163_type, _seq); p->level--; return _seq; } @@ -34216,7 +34050,6 @@ _loop0_164_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -34267,7 +34100,6 @@ _loop0_164_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_164_type, _seq); p->level--; return _seq; } @@ -34286,7 +34118,6 @@ _loop1_165_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -34342,7 +34173,6 @@ _loop1_165_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_165_type, _seq); p->level--; return _seq; } @@ -34419,7 +34249,6 @@ _loop0_167_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -34470,7 +34299,6 @@ _loop0_167_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_167_type, _seq); p->level--; return _seq; } @@ -34547,7 +34375,6 @@ _loop0_169_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -34598,7 +34425,6 @@ _loop0_169_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_169_type, _seq); p->level--; return _seq; } @@ -34617,7 +34443,6 @@ _loop1_170_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -34673,7 +34498,6 @@ _loop1_170_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_170_type, _seq); p->level--; return _seq; } @@ -34869,7 +34693,6 @@ _loop0_174_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -34920,7 +34743,6 @@ _loop0_174_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_174_type, _seq); p->level--; return _seq; } @@ -35074,7 +34896,6 @@ _loop1_177_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -35130,7 +34951,6 @@ _loop1_177_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_177_type, _seq); p->level--; return _seq; } @@ -35207,7 +35027,6 @@ _loop0_179_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -35258,7 +35077,6 @@ _loop0_179_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_179_type, _seq); p->level--; return _seq; } @@ -35277,7 +35095,6 @@ _loop0_180_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -35328,7 +35145,6 @@ _loop0_180_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_180_type, _seq); p->level--; return _seq; } @@ -35347,7 +35163,6 @@ _loop0_181_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -35398,7 +35213,6 @@ _loop0_181_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_181_type, _seq); p->level--; return _seq; } @@ -35417,7 +35231,6 @@ _loop0_183_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -35477,7 +35290,6 @@ _loop0_183_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_183_type, _seq); p->level--; return _seq; } @@ -35596,7 +35408,6 @@ _loop0_185_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -35647,7 +35458,6 @@ _loop0_185_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_185_type, _seq); p->level--; return _seq; } @@ -35724,7 +35534,6 @@ _loop0_187_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -35775,7 +35584,6 @@ _loop0_187_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_187_type, _seq); p->level--; return _seq; } @@ -35794,7 +35602,6 @@ _loop1_188_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -35850,7 +35657,6 @@ _loop1_188_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_188_type, _seq); p->level--; return _seq; } @@ -35869,7 +35675,6 @@ _loop1_189_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -35925,7 +35730,6 @@ _loop1_189_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_189_type, _seq); p->level--; return _seq; } @@ -36063,7 +35867,6 @@ _loop0_192_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -36114,7 +35917,6 @@ _loop0_192_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_192_type, _seq); p->level--; return _seq; } @@ -36345,7 +36147,6 @@ _loop0_197_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -36405,7 +36206,6 @@ _loop0_197_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_197_type, _seq); p->level--; return _seq; } @@ -36466,7 +36266,6 @@ _loop0_199_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -36526,7 +36325,6 @@ _loop0_199_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_199_type, _seq); p->level--; return _seq; } @@ -36587,7 +36385,6 @@ _loop0_201_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -36647,7 +36444,6 @@ _loop0_201_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_201_type, _seq); p->level--; return _seq; } @@ -36708,7 +36504,6 @@ _loop0_203_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -36768,7 +36563,6 @@ _loop0_203_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_203_type, _seq); p->level--; return _seq; } @@ -36887,7 +36681,6 @@ _loop0_205_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -36938,7 +36731,6 @@ _loop0_205_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_205_type, _seq); p->level--; return _seq; } @@ -36957,7 +36749,6 @@ _loop1_206_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -37013,7 +36804,6 @@ _loop1_206_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_206_type, _seq); p->level--; return _seq; } @@ -37074,7 +36864,6 @@ _loop0_208_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -37125,7 +36914,6 @@ _loop0_208_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_208_type, _seq); p->level--; return _seq; } @@ -37144,7 +36932,6 @@ _loop1_209_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -37200,7 +36987,6 @@ _loop1_209_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_209_type, _seq); p->level--; return _seq; } @@ -37664,7 +37450,6 @@ _loop0_221_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -37724,7 +37509,6 @@ _loop0_221_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_221_type, _seq); p->level--; return _seq; } diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py index c41d87b74d1c494..e72ce7afdc4796f 100644 --- a/Tools/peg_generator/pegen/c_generator.py +++ b/Tools/peg_generator/pegen/c_generator.py @@ -619,7 +619,8 @@ def _handle_loop_rule_body(self, node: Rule, rhs: Rhs) -> None: self.add_return("_res") self.print("}") self.print("int _mark = p->mark;") - self.print("int _start_mark = p->mark;") + if memoize: + self.print("int _start_mark = p->mark;") self.print("void **_children = PyMem_Malloc(sizeof(void *));") self.out_of_memory_return(f"!_children") self.print("Py_ssize_t _children_capacity = 1;") @@ -642,7 +643,7 @@ def _handle_loop_rule_body(self, node: Rule, rhs: Rhs) -> None: self.out_of_memory_return(f"!_seq", cleanup_code="PyMem_Free(_children);") self.print("for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]);") self.print("PyMem_Free(_children);") - if node.name: + if memoize and node.name: self.print(f"_PyPegen_insert_memo(p, _start_mark, {node.name}_type, _seq);") self.add_return("_seq") From d3ca042c99b2e86a2bb927a877fdfcbacdc22f89 Mon Sep 17 00:00:00 2001 From: Hyunkyun Moon Date: Mon, 6 Mar 2023 22:56:19 +0900 Subject: [PATCH 20/97] gh-95672: Fix versionadded indentation of get_pagesize in test.rst (gh-102455) --- Doc/library/test.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 3c759648e4e626b..c60b4da75d1acb6 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -540,7 +540,7 @@ The :mod:`test.support` module defines the following functions: Get size of a page in bytes. - .. versionadded:: 3.12 + .. versionadded:: 3.12 .. function:: setswitchinterval(interval) From f105fe4f0a704bedee21d95f1a08bc14a1fcea2a Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Mon, 6 Mar 2023 17:49:31 +0000 Subject: [PATCH 21/97] gh-102192: Replace PyErr_Fetch/Restore etc by more efficient alternatives in sub interpreters module (#102472) --- Modules/_xxsubinterpretersmodule.c | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 461c505c092c701..79dbe3474ba9e81 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -61,9 +61,9 @@ add_new_exception(PyObject *mod, const char *name, PyObject *base) static int _release_xid_data(_PyCrossInterpreterData *data, int ignoreexc) { - PyObject *exctype, *excval, *exctb; + PyObject *exc; if (ignoreexc) { - PyErr_Fetch(&exctype, &excval, &exctb); + exc = PyErr_GetRaisedException(); } int res = _PyCrossInterpreterData_Release(data); if (res < 0) { @@ -84,7 +84,7 @@ _release_xid_data(_PyCrossInterpreterData *data, int ignoreexc) } } if (ignoreexc) { - PyErr_Restore(exctype, excval, exctb); + PyErr_SetRaisedException(exc); } return res; } @@ -294,9 +294,9 @@ _sharedexception_free(_sharedexception *exc) } static _sharedexception * -_sharedexception_bind(PyObject *exctype, PyObject *exc, PyObject *tb) +_sharedexception_bind(PyObject *exc) { - assert(exctype != NULL); + assert(exc != NULL); char *failure = NULL; _sharedexception *err = _sharedexception_new(); @@ -304,7 +304,7 @@ _sharedexception_bind(PyObject *exctype, PyObject *exc, PyObject *tb) goto finally; } - PyObject *name = PyUnicode_FromFormat("%S", exctype); + PyObject *name = PyUnicode_FromFormat("%S", Py_TYPE(exc)); if (name == NULL) { failure = "unable to format exception type name"; goto finally; @@ -432,10 +432,7 @@ static int _run_script(PyInterpreterState *interp, const char *codestr, _sharedns *shared, _sharedexception **exc) { - PyObject *exctype = NULL; PyObject *excval = NULL; - PyObject *tb = NULL; - PyObject *main_mod = _PyInterpreterState_GetMainModule(interp); if (main_mod == NULL) { goto error; @@ -469,12 +466,9 @@ _run_script(PyInterpreterState *interp, const char *codestr, return 0; error: - PyErr_Fetch(&exctype, &excval, &tb); - - _sharedexception *sharedexc = _sharedexception_bind(exctype, excval, tb); - Py_XDECREF(exctype); + excval = PyErr_GetRaisedException(); + _sharedexception *sharedexc = _sharedexception_bind(excval); Py_XDECREF(excval); - Py_XDECREF(tb); if (sharedexc == NULL) { fprintf(stderr, "RunFailedError: script raised an uncaught exception"); PyErr_Clear(); From d959bcd4a0393a120fa12c034de4041037d171c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Lapeyre?= Date: Mon, 6 Mar 2023 22:20:52 +0100 Subject: [PATCH 22/97] Add gettext support to tools/extensions/c_annotations.py (#101989) --- Doc/tools/extensions/c_annotations.py | 7 ++++--- Doc/tools/templates/dummy.html | 5 +++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Doc/tools/extensions/c_annotations.py b/Doc/tools/extensions/c_annotations.py index e5dc82cf0dc2d9d..5af56433f41573a 100644 --- a/Doc/tools/extensions/c_annotations.py +++ b/Doc/tools/extensions/c_annotations.py @@ -24,6 +24,7 @@ from docutils.parsers.rst import directives from docutils.parsers.rst import Directive from docutils.statemachine import StringList +from sphinx.locale import _ as sphinx_gettext import csv from sphinx import addnodes @@ -168,11 +169,11 @@ def add_annotations(self, app, doctree): elif not entry.result_type.endswith("Object*"): continue if entry.result_refs is None: - rc = 'Return value: Always NULL.' + rc = sphinx_gettext('Return value: Always NULL.') elif entry.result_refs: - rc = 'Return value: New reference.' + rc = sphinx_gettext('Return value: New reference.') else: - rc = 'Return value: Borrowed reference.' + rc = sphinx_gettext('Return value: Borrowed reference.') node.insert(0, nodes.emphasis(rc, rc, classes=['refcount'])) diff --git a/Doc/tools/templates/dummy.html b/Doc/tools/templates/dummy.html index 3438b44377fcb96..bab4aaeb4604b8f 100644 --- a/Doc/tools/templates/dummy.html +++ b/Doc/tools/templates/dummy.html @@ -7,6 +7,11 @@ {% trans %}Deprecated since version {deprecated}, will be removed in version {removed}{% endtrans %} {% trans %}Deprecated since version {deprecated}, removed in version {removed}{% endtrans %} +In extensions/c_annotations.py: + +{% trans %}Return value: Always NULL.{% endtrans %} +{% trans %}Return value: New reference.{% endtrans %} +{% trans %}Return value: Borrowed reference.{% endtrans %} In docsbuild-scripts, when rewriting indexsidebar.html with actual versions: From c84e6f32df989908685ea8b6cd49ddde9f428524 Mon Sep 17 00:00:00 2001 From: JosephSBoyle <48555120+JosephSBoyle@users.noreply.github.com> Date: Mon, 6 Mar 2023 22:02:19 +0000 Subject: [PATCH 23/97] Remove unused import of `warnings` from `unittest.loader` (#102479) --- Lib/unittest/loader.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/unittest/loader.py b/Lib/unittest/loader.py index 80d4fbdd8e36062..b989284a640e141 100644 --- a/Lib/unittest/loader.py +++ b/Lib/unittest/loader.py @@ -6,7 +6,6 @@ import traceback import types import functools -import warnings from fnmatch import fnmatch, fnmatchcase From f9cdeb7b99d408a2e884101ede576952510bcc9b Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Mon, 6 Mar 2023 20:02:24 -0500 Subject: [PATCH 24/97] gh-101759: Update macOS installer SQLite 3.40.1 checksum (gh-102485) --- Mac/BuildScript/build-installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 5ea2cfc5183ef50..63fa21b2b33d174 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -361,7 +361,7 @@ def library_recipes(): dict( name="SQLite 3.40.1", url="https://sqlite.org/2022/sqlite-autoconf-3400100.tar.gz", - checksum="5498af3a357753d473ee713e363fa5b7", + checksum="42175b1a1d23529cb133bbd2b5900afd", extra_cflags=('-Os ' '-DSQLITE_ENABLE_FTS5 ' '-DSQLITE_ENABLE_FTS4 ' From 8606697f49dc58ff7e18147401ac65a09c38cf57 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 6 Mar 2023 19:40:09 -0700 Subject: [PATCH 25/97] gh-90110: Fix the c-analyzer Tool (#102483) Some incompatible changes had gone in, and the "ignore" lists weren't properly undated. This change fixes that. It's necessary prior to enabling test_check_c_globals, which I hope to do soon. Note that this does include moving last_resort_memory_error to PyInterpreterState. https://github.com/python/cpython/issues/90110 --- Include/internal/pycore_global_objects.h | 1 + Include/internal/pycore_intrinsics.h | 4 +-- Include/internal/pycore_runtime_init.h | 6 ++++ Objects/exceptions.c | 13 +++----- Python/intrinsics.c | 4 +-- Tools/c-analyzer/c_parser/parser/_common.py | 22 ++++++++++---- .../c-analyzer/c_parser/parser/_func_body.py | 25 +++++++++------- Tools/c-analyzer/c_parser/preprocessor/gcc.py | 3 +- Tools/c-analyzer/cpython/_parser.py | 9 +++++- Tools/c-analyzer/cpython/globals-to-fix.tsv | 2 +- Tools/c-analyzer/cpython/ignored.tsv | 30 +++++++++++++++++-- 11 files changed, 85 insertions(+), 34 deletions(-) diff --git a/Include/internal/pycore_global_objects.h b/Include/internal/pycore_global_objects.h index 30c7c4e3bbd067b..9957da1fc5f22ad 100644 --- a/Include/internal/pycore_global_objects.h +++ b/Include/internal/pycore_global_objects.h @@ -86,6 +86,7 @@ struct _Py_interp_static_objects { // hamt_empty is here instead of global because of its weakreflist. _PyGC_Head_UNUSED _hamt_empty_gc_not_used; PyHamtObject hamt_empty; + PyBaseExceptionObject last_resort_memory_error; } singletons; }; diff --git a/Include/internal/pycore_intrinsics.h b/Include/internal/pycore_intrinsics.h index deac145fff7627e..46a52740eb8a0c3 100644 --- a/Include/internal/pycore_intrinsics.h +++ b/Include/internal/pycore_intrinsics.h @@ -21,6 +21,6 @@ typedef PyObject *(*instrinsic_func1)(PyThreadState* tstate, PyObject *value); typedef PyObject *(*instrinsic_func2)(PyThreadState* tstate, PyObject *value1, PyObject *value2); -extern instrinsic_func1 _PyIntrinsics_UnaryFunctions[]; -extern instrinsic_func2 _PyIntrinsics_BinaryFunctions[]; +extern const instrinsic_func1 _PyIntrinsics_UnaryFunctions[]; +extern const instrinsic_func2 _PyIntrinsics_BinaryFunctions[]; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index b54adf04761d4eb..a2cc7c87c2f3e29 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -14,6 +14,9 @@ extern "C" { #include "pycore_obmalloc_init.h" +extern PyTypeObject _PyExc_MemoryError; + + /* The static initializers defined here should only be used in the runtime init code (in pystate.c and pylifecycle.c). */ @@ -120,6 +123,9 @@ extern "C" { .ob_base = _PyObject_IMMORTAL_INIT(&_PyHamt_Type), \ .h_root = (PyHamtNode*)&_Py_SINGLETON(hamt_bitmap_node_empty), \ }, \ + .last_resort_memory_error = { \ + _PyObject_IMMORTAL_INIT(&_PyExc_MemoryError), \ + }, \ }, \ }, \ ._initial_thread = _PyThreadState_INIT, \ diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 976f84dbf63c933..a473cbdfeda7fc1 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -3207,8 +3207,6 @@ SimpleExtendsException(PyExc_Exception, ReferenceError, #define MEMERRORS_SAVE 16 -static PyBaseExceptionObject last_resort_memory_error; - static PyObject * get_memory_error(int allow_allocation, PyObject *args, PyObject *kwds) { @@ -3216,7 +3214,9 @@ get_memory_error(int allow_allocation, PyObject *args, PyObject *kwds) struct _Py_exc_state *state = get_exc_state(); if (state->memerrors_freelist == NULL) { if (!allow_allocation) { - return Py_NewRef(&last_resort_memory_error); + PyInterpreterState *interp = _PyInterpreterState_GET(); + return Py_NewRef( + &_Py_INTERP_SINGLETON(interp, last_resort_memory_error)); } PyObject *result = BaseException_new((PyTypeObject *)PyExc_MemoryError, args, kwds); return result; @@ -3239,8 +3239,6 @@ get_memory_error(int allow_allocation, PyObject *args, PyObject *kwds) return (PyObject *)self; } -static PyBaseExceptionObject last_resort_memory_error; - static PyObject * MemoryError_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { @@ -3325,7 +3323,7 @@ free_preallocated_memerrors(struct _Py_exc_state *state) } -static PyTypeObject _PyExc_MemoryError = { +PyTypeObject _PyExc_MemoryError = { PyVarObject_HEAD_INIT(NULL, 0) "MemoryError", sizeof(PyBaseExceptionObject), @@ -3339,9 +3337,6 @@ static PyTypeObject _PyExc_MemoryError = { }; PyObject *PyExc_MemoryError = (PyObject *) &_PyExc_MemoryError; -static PyBaseExceptionObject last_resort_memory_error = { - _PyObject_IMMORTAL_INIT(&_PyExc_MemoryError) -}; /* * BufferError extends Exception diff --git a/Python/intrinsics.c b/Python/intrinsics.c index 9e90ef32130f1d9..cca29d859902a4a 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -199,7 +199,7 @@ list_to_tuple(PyThreadState* unused, PyObject *v) return _PyTuple_FromArray(((PyListObject *)v)->ob_item, Py_SIZE(v)); } -instrinsic_func1 +const instrinsic_func1 _PyIntrinsics_UnaryFunctions[] = { [0] = no_intrinsic, [INTRINSIC_PRINT] = print_expr, @@ -221,7 +221,7 @@ prep_reraise_star(PyThreadState* unused, PyObject *orig, PyObject *excs) return _PyExc_PrepReraiseStar(orig, excs); } -instrinsic_func2 +const instrinsic_func2 _PyIntrinsics_BinaryFunctions[] = { [INTRINSIC_PREP_RERAISE_STAR] = prep_reraise_star, }; diff --git a/Tools/c-analyzer/c_parser/parser/_common.py b/Tools/c-analyzer/c_parser/parser/_common.py index d468d5442a939f0..2eacace2c001df9 100644 --- a/Tools/c-analyzer/c_parser/parser/_common.py +++ b/Tools/c-analyzer/c_parser/parser/_common.py @@ -7,13 +7,25 @@ ) -def log_match(group, m): +def log_match(group, m, depth_before=None, depth_after=None): from . import _logger - text = m.group(0) - if text.startswith(('(', ')')) or text.endswith(('(', ')')): - _logger.debug(f'matched <{group}> ({text!r})') + + if m is not None: + text = m.group(0) + if text.startswith(('(', ')')) or text.endswith(('(', ')')): + _logger.debug(f'matched <{group}> ({text!r})') + else: + _logger.debug(f'matched <{group}> ({text})') + + elif depth_before is not None or depth_after is not None: + if depth_before is None: + depth_before = '???' + elif depth_after is None: + depth_after = '???' + _logger.log(1, f'depth: %s -> %s', depth_before, depth_after) + else: - _logger.debug(f'matched <{group}> ({text})') + raise NotImplementedError('this should not have been hit') ############################# diff --git a/Tools/c-analyzer/c_parser/parser/_func_body.py b/Tools/c-analyzer/c_parser/parser/_func_body.py index 42fd459e111d2ce..25f2f5807ae8279 100644 --- a/Tools/c-analyzer/c_parser/parser/_func_body.py +++ b/Tools/c-analyzer/c_parser/parser/_func_body.py @@ -65,11 +65,11 @@ def parse_function_body(name, text, resolve, source, anon_name, parent): ) = m.groups() if empty: - log_match('', m) + log_match('', m, depth) resolve(None, None, None, text) yield None, text elif inline_kind: - log_match('', m) + log_match('', m, depth) kind = inline_kind name = inline_name or anon_name('inline-') data = [] # members @@ -92,7 +92,7 @@ def parse_function_body(name, text, resolve, source, anon_name, parent): # XXX Should "parent" really be None for inline type decls? yield resolve(kind, data, name, text, None), text elif block_close: - log_match('', m) + log_match('', m, depth) depth -= 1 resolve(None, None, None, text) # XXX This isn't great. Calling resolve() should have @@ -101,13 +101,13 @@ def parse_function_body(name, text, resolve, source, anon_name, parent): # needs to be fixed. yield None, text elif compound_bare: - log_match('', m) + log_match('', m, depth) yield resolve('statement', compound_bare, None, text, parent), text elif compound_labeled: - log_match('', m) + log_match('', m, depth) yield resolve('statement', compound_labeled, None, text, parent), text elif compound_paren: - log_match('', m) + log_match('', m, depth) try: pos = match_paren(text) except ValueError: @@ -132,7 +132,7 @@ def parse_function_body(name, text, resolve, source, anon_name, parent): } yield resolve('statement', data, None, text, parent), text elif block_open: - log_match('', m) + log_match('', m, depth) depth += 1 if block_leading: # An inline block: the last evaluated expression is used @@ -144,10 +144,10 @@ def parse_function_body(name, text, resolve, source, anon_name, parent): resolve(None, None, None, text) yield None, text elif simple_ending: - log_match('', m) + log_match('', m, depth) yield resolve('statement', simple_stmt, None, text, parent), text elif var_ending: - log_match('', m) + log_match('', m, depth) kind = 'variable' _, name, vartype = parse_var_decl(decl) data = { @@ -220,7 +220,7 @@ def _parse_next_local_static(m, srcinfo, anon_name, func, depth): remainder = srcinfo.text[m.end():] if inline_kind: - log_match('func inline', m) + log_match('func inline', m, depth, depth) kind = inline_kind name = inline_name or anon_name('inline-') # Immediately emit a forward declaration. @@ -249,7 +249,7 @@ def parse_body(source): yield parse_body, depth elif static_decl: - log_match('local variable', m) + log_match('local variable', m, depth, depth) _, name, data = parse_var_decl(static_decl) yield srcinfo.resolve('variable', data, name, parent=func), depth @@ -266,10 +266,13 @@ def parse_body(source): else: log_match('func other', m) if block_open: + log_match('func other', None, depth, depth + 1) depth += 1 elif block_close: + log_match('func other', None, depth, depth - 1) depth -= 1 elif stmt_end: + log_match('func other', None, depth, depth) pass else: # This should be unreachable. diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index 770802253792d8f..7ef1a8afc3b135e 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -29,7 +29,7 @@ [^()]* )* ) # - ( [)] [)] )? # + ( [)] [)] ) # ''', re.VERBOSE) POST_ARGS = ( @@ -156,6 +156,7 @@ def _iter_top_include_lines(lines, topfile, cwd, if name != 'pragma': raise Exception(line) else: + line = re.sub(r'__inline__', 'inline', line) if not raw: line, partial = _strip_directives(line, partial=partial) yield _common.SourceLine( diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index ab1d6257f1b1a94..e7764165d36c4c0 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -105,9 +105,14 @@ def clean_lines(text): * ./Include * ./Include/internal +Modules/_decimal/**/*.c Modules/_decimal/libmpdec +Modules/_hacl/*.c Modules/_hacl/include +Modules/_hacl/*.h Modules/_hacl/include Modules/_tkinter.c /usr/include/tcl8.6 +Modules/md5module.c Modules/_hacl/include +Modules/sha1module.c Modules/_hacl/include +Modules/sha2module.c Modules/_hacl/include Modules/tkappinit.c /usr/include/tcl -Modules/_decimal/**/*.c Modules/_decimal/libmpdec Objects/stringlib/*.h Objects # @end=tsv@ @@ -173,6 +178,7 @@ def clean_lines(text): Modules/_functoolsmodule.c Py_BUILD_CORE 1 Modules/_heapqmodule.c Py_BUILD_CORE 1 Modules/_io/*.c Py_BUILD_CORE 1 +Modules/_io/*.h Py_BUILD_CORE 1 Modules/_localemodule.c Py_BUILD_CORE 1 Modules/_operator.c Py_BUILD_CORE 1 Modules/_posixsubprocess.c Py_BUILD_CORE 1 @@ -296,6 +302,7 @@ def clean_lines(text): # First match wins. _abs('Modules/_ctypes/ctypes.h'): (5_000, 500), _abs('Modules/_datetimemodule.c'): (20_000, 300), + _abs('Modules/_hacl/*.c'): (200_000, 500), _abs('Modules/posixmodule.c'): (20_000, 500), _abs('Modules/termios.c'): (10_000, 800), _abs('Modules/_testcapimodule.c'): (20_000, 400), diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 2e28c50c6ff69ae..57b8542fb46482d 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -539,7 +539,7 @@ Modules/_tkinter.c - command_mutex - Modules/_tkinter.c - HeadFHCD - Modules/_tkinter.c - stdin_ready - Modules/_tkinter.c - event_tstate - -Modules/_xxsubinterpretersmodule.c - _globals - +Modules/_xxinterpchannelsmodule.c - _globals - Modules/readline.c - completer_word_break_characters - Modules/readline.c - _history_length - Modules/readline.c - should_auto_add_history - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 849e20a1b0f4ebe..700ddf2851839e1 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -206,6 +206,10 @@ Modules/_decimal/_decimal.c - invalid_signals_err - Modules/_decimal/_decimal.c - signal_map - Modules/_decimal/_decimal.c - ssize_constants - Modules/_elementtree.c - ExpatMemoryHandler - +Modules/_hashopenssl.c - py_hashes - +Modules/_hacl/Hacl_Hash_SHA1.c - _h0 - +Modules/_hacl/Hacl_Hash_MD5.c - _h0 - +Modules/_hacl/Hacl_Hash_MD5.c - _t - Modules/_io/_iomodule.c - static_types - Modules/_io/textio.c - encodefuncs - Modules/_io/winconsoleio.c - _PyWindowsConsoleIO_Type - @@ -218,9 +222,10 @@ Modules/_sre/sre_targets.h - sre_targets - Modules/_sre.c pattern_repr flag_names - Modules/_struct.c - bigendian_table - Modules/_struct.c - lilendian_table - +Modules/_struct.c - native_table - Modules/_tkinter.c - state_key - -Modules/_xxsubinterpretersmodule.c - _channelid_end_recv - -Modules/_xxsubinterpretersmodule.c - _channelid_end_send - +Modules/_xxinterpchannelsmodule.c - _channelid_end_recv - +Modules/_xxinterpchannelsmodule.c - _channelid_end_send - Modules/_zoneinfo.c - DAYS_BEFORE_MONTH - Modules/_zoneinfo.c - DAYS_IN_MONTH - Modules/arraymodule.c - descriptors - @@ -332,12 +337,15 @@ Python/frozen.c - _PyImport_FrozenTest - Python/getopt.c - longopts - Python/import.c - _PyImport_Inittab - Python/import.c - _PySys_ImplCacheTag - +Python/intrinsics.c - _PyIntrinsics_UnaryFunctions - +Python/intrinsics.c - _PyIntrinsics_BinaryFunctions - Python/opcode_targets.h - opcode_targets - Python/perf_trampoline.c - _Py_perfmap_callbacks - Python/pyhash.c - PyHash_Func - Python/pylifecycle.c - _C_LOCALE_WARNING - Python/pylifecycle.c - _PyOS_mystrnicmp_hack - Python/pylifecycle.c - _TARGET_LOCALES - +Python/pylifecycle.c - INTERPRETER_TRAMPOLINE_CODEDEF - Python/pystate.c - initial - Python/specialize.c - adaptive_opcodes - Python/specialize.c - cache_requirements - @@ -392,8 +400,23 @@ Modules/_testbuffer.c ndarray_memoryview_from_buffer strides - Modules/_testbuffer.c ndarray_memoryview_from_buffer suboffsets - Modules/_testbuffer.c ndarray_push kwlist - Modules/_testbuffer.c staticarray_init kwlist - +Modules/_testcapi/code.c get_code_extra_index key - +Modules/_testcapi/datetime.c - test_run_counter - +Modules/_testcapi/exceptions.c - PyRecursingInfinitelyError_Type - Modules/_testcapi/heaptype.c - _testcapimodule - +Modules/_testcapi/mem.c - FmData - +Modules/_testcapi/mem.c - FmHook - +Modules/_testcapi/structmember.c - test_structmembersType_OldAPI - Modules/_testcapi/unicode.c - _testcapimodule - +Modules/_testcapi/watchers.c - g_dict_watch_events - +Modules/_testcapi/watchers.c - g_dict_watchers_installed - +Modules/_testcapi/watchers.c - g_type_modified_events - +Modules/_testcapi/watchers.c - g_type_watchers_installed - +Modules/_testcapi/watchers.c - num_code_object_created_events - +Modules/_testcapi/watchers.c - num_code_object_destroyed_events - +Modules/_testcapi/watchers.c - pyfunc_watchers - +Modules/_testcapi/watchers.c - func_watcher_ids - +Modules/_testcapi/watchers.c - func_watcher_callbacks - Modules/_testcapimodule.c - ContainerNoGC_members - Modules/_testcapimodule.c - ContainerNoGC_type - Modules/_testcapimodule.c - FmData - @@ -460,11 +483,13 @@ Modules/_testcapimodule.c - meth_static_methods - Modules/_testcapimodule.c - ml - Modules/_testcapimodule.c - str1 - Modules/_testcapimodule.c - str2 - +Modules/_testcapimodule.c - test_c_thread - Modules/_testcapimodule.c - test_members - Modules/_testcapimodule.c - test_run_counter - Modules/_testcapimodule.c - test_structmembersType - Modules/_testcapimodule.c - thread_done - Modules/_testcapimodule.c - x - +Modules/_testcapimodule.c - wait_done - Modules/_testcapimodule.c getargs_keyword_only keywords - Modules/_testcapimodule.c getargs_keywords keywords - Modules/_testcapimodule.c getargs_positional_only_and_keywords keywords - @@ -526,6 +551,7 @@ Modules/_testmultiphase.c - slots_exec_unreported_exception - Modules/_testmultiphase.c - slots_nonmodule_with_exec_slots - Modules/_testmultiphase.c - testexport_methods - Modules/_testmultiphase.c - uninitialized_def - +Modules/_testsinglephase.c - global_state - Modules/_xxtestfuzz/_xxtestfuzz.c - _fuzzmodule - Modules/_xxtestfuzz/_xxtestfuzz.c - module_methods - Modules/_xxtestfuzz/fuzzer.c - SRE_FLAG_DEBUG - From 80b19a30c0d5f9f8a8651e7f8847c0e68671c89a Mon Sep 17 00:00:00 2001 From: "C.A.M. Gerlach" Date: Mon, 6 Mar 2023 20:45:52 -0600 Subject: [PATCH 26/97] gh-95913: Edit Faster CPython section in 3.11 WhatsNew (GH-98429) Co-authored-by: C.A.M. Gerlach --- Doc/whatsnew/3.11.rst | 186 +++++++++++++++++++++++++----------------- 1 file changed, 109 insertions(+), 77 deletions(-) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index bffb8d03aa7cbbe..391ea531281a483 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -1317,14 +1317,17 @@ This section covers specific optimizations independent of the Faster CPython ============== -CPython 3.11 is on average `25% faster `_ -than CPython 3.10 when measured with the +CPython 3.11 is an average of +`25% faster `_ +than CPython 3.10 as measured with the `pyperformance `_ benchmark suite, -and compiled with GCC on Ubuntu Linux. Depending on your workload, the speedup -could be up to 10-60% faster. +when compiled with GCC on Ubuntu Linux. +Depending on your workload, the overall speedup could be 10-60%. -This project focuses on two major areas in Python: faster startup and faster -runtime. Other optimizations not under this project are listed in `Optimizations`_. +This project focuses on two major areas in Python: +:ref:`whatsnew311-faster-startup` and :ref:`whatsnew311-faster-runtime`. +Optimizations not covered by this project are listed separately under +:ref:`whatsnew311-optimizations`. .. _whatsnew311-faster-startup: @@ -1337,8 +1340,8 @@ Faster Startup Frozen imports / Static code objects ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Python caches bytecode in the :ref:`__pycache__` directory to -speed up module loading. +Python caches :term:`bytecode` in the :ref:`__pycache__ ` +directory to speed up module loading. Previously in 3.10, Python module execution looked like this: @@ -1347,8 +1350,9 @@ Previously in 3.10, Python module execution looked like this: Read __pycache__ -> Unmarshal -> Heap allocated code object -> Evaluate In Python 3.11, the core modules essential for Python startup are "frozen". -This means that their code objects (and bytecode) are statically allocated -by the interpreter. This reduces the steps in module execution process to this: +This means that their :ref:`codeobjects` (and bytecode) +are statically allocated by the interpreter. +This reduces the steps in module execution process to: .. code-block:: text @@ -1357,7 +1361,7 @@ by the interpreter. This reduces the steps in module execution process to this: Interpreter startup is now 10-15% faster in Python 3.11. This has a big impact for short-running programs using Python. -(Contributed by Eric Snow, Guido van Rossum and Kumar Aditya in numerous issues.) +(Contributed by Eric Snow, Guido van Rossum and Kumar Aditya in many issues.) .. _whatsnew311-faster-runtime: @@ -1370,17 +1374,19 @@ Faster Runtime Cheaper, lazy Python frames ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Python frames are created whenever Python calls a Python function. This frame -holds execution information. The following are new frame optimizations: +Python frames, holding execution information, +are created whenever Python calls a Python function. +The following are new frame optimizations: - Streamlined the frame creation process. - Avoided memory allocation by generously re-using frame space on the C stack. - Streamlined the internal frame struct to contain only essential information. Frames previously held extra debugging and memory management information. -Old-style frame objects are now created only when requested by debuggers or -by Python introspection functions such as ``sys._getframe`` or -``inspect.currentframe``. For most user code, no frame objects are +Old-style :ref:`frame objects ` +are now created only when requested by debuggers +or by Python introspection functions such as :func:`sys._getframe` and +:func:`inspect.currentframe`. For most user code, no frame objects are created at all. As a result, nearly all Python functions calls have sped up significantly. We measured a 3-7% speedup in pyperformance. @@ -1401,10 +1407,11 @@ In 3.11, when CPython detects Python code calling another Python function, it sets up a new frame, and "jumps" to the new code inside the new frame. This avoids calling the C interpreting function altogether. -Most Python function calls now consume no C stack space. This speeds up -most of such calls. In simple recursive functions like fibonacci or -factorial, a 1.7x speedup was observed. This also means recursive functions -can recurse significantly deeper (if the user increases the recursion limit). +Most Python function calls now consume no C stack space, speeding them up. +In simple recursive functions like fibonacci or +factorial, we observed a 1.7x speedup. This also means recursive functions +can recurse significantly deeper +(if the user increases the recursion limit with :func:`sys.setrecursionlimit`). We measured a 1-3% improvement in pyperformance. (Contributed by Pablo Galindo and Mark Shannon in :issue:`45256`.) @@ -1415,7 +1422,7 @@ We measured a 1-3% improvement in pyperformance. PEP 659: Specializing Adaptive Interpreter ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -:pep:`659` is one of the key parts of the faster CPython project. The general +:pep:`659` is one of the key parts of the Faster CPython project. The general idea is that while Python is a dynamic language, most code has regions where objects and types rarely change. This concept is known as *type stability*. @@ -1424,17 +1431,18 @@ in the executing code. Python will then replace the current operation with a more specialized one. This specialized operation uses fast paths available only to those use cases/types, which generally outperform their generic counterparts. This also brings in another concept called *inline caching*, where -Python caches the results of expensive operations directly in the bytecode. +Python caches the results of expensive operations directly in the +:term:`bytecode`. The specializer will also combine certain common instruction pairs into one -superinstruction. This reduces the overhead during execution. +superinstruction, reducing the overhead during execution. Python will only specialize when it sees code that is "hot" (executed multiple times). This prevents Python -from wasting time for run-once code. Python can also de-specialize when code is +from wasting time on run-once code. Python can also de-specialize when code is too dynamic or when the use changes. Specialization is attempted periodically, -and specialization attempts are not too expensive. This allows specialization -to adapt to new circumstances. +and specialization attempts are not too expensive, +allowing specialization to adapt to new circumstances. (PEP written by Mark Shannon, with ideas inspired by Stefan Brunthaler. See :pep:`659` for more information. Implementation by Mark Shannon and Brandt @@ -1447,32 +1455,32 @@ Bucher, with additional help from Irit Katriel and Dennis Sweeney.) | Operation | Form | Specialization | Operation speedup | Contributor(s) | | | | | (up to) | | +===============+====================+=======================================================+===================+===================+ -| Binary | ``x+x; x*x; x-x;`` | Binary add, multiply and subtract for common types | 10% | Mark Shannon, | -| operations | | such as ``int``, ``float``, and ``str`` take custom | | Dong-hee Na, | -| | | fast paths for their underlying types. | | Brandt Bucher, | +| Binary | ``x + x`` | Binary add, multiply and subtract for common types | 10% | Mark Shannon, | +| operations | | such as :class:`int`, :class:`float` and :class:`str` | | Dong-hee Na, | +| | ``x - x`` | take custom fast paths for their underlying types. | | Brandt Bucher, | | | | | | Dennis Sweeney | +| | ``x * x`` | | | | +---------------+--------------------+-------------------------------------------------------+-------------------+-------------------+ -| Subscript | ``a[i]`` | Subscripting container types such as ``list``, | 10-25% | Irit Katriel, | -| | | ``tuple`` and ``dict`` directly index the underlying | | Mark Shannon | -| | | data structures. | | | +| Subscript | ``a[i]`` | Subscripting container types such as :class:`list`, | 10-25% | Irit Katriel, | +| | | :class:`tuple` and :class:`dict` directly index | | Mark Shannon | +| | | the underlying data structures. | | | | | | | | | -| | | Subscripting custom ``__getitem__`` | | | +| | | Subscripting custom :meth:`~object.__getitem__` | | | | | | is also inlined similar to :ref:`inline-calls`. | | | +---------------+--------------------+-------------------------------------------------------+-------------------+-------------------+ | Store | ``a[i] = z`` | Similar to subscripting specialization above. | 10-25% | Dennis Sweeney | | subscript | | | | | +---------------+--------------------+-------------------------------------------------------+-------------------+-------------------+ | Calls | ``f(arg)`` | Calls to common builtin (C) functions and types such | 20% | Mark Shannon, | -| | ``C(arg)`` | as ``len`` and ``str`` directly call their underlying | | Ken Jin | -| | | C version. This avoids going through the internal | | | -| | | calling convention. | | | -| | | | | | +| | | as :func:`len` and :class:`str` directly call their | | Ken Jin | +| | ``C(arg)`` | underlying C version. This avoids going through the | | | +| | | internal calling convention. | | | +---------------+--------------------+-------------------------------------------------------+-------------------+-------------------+ -| Load | ``print`` | The object's index in the globals/builtins namespace | [1]_ | Mark Shannon | -| global | ``len`` | is cached. Loading globals and builtins require | | | -| variable | | zero namespace lookups. | | | +| Load | ``print`` | The object's index in the globals/builtins namespace | [#load-global]_ | Mark Shannon | +| global | | is cached. Loading globals and builtins require | | | +| variable | ``len`` | zero namespace lookups. | | | +---------------+--------------------+-------------------------------------------------------+-------------------+-------------------+ -| Load | ``o.attr`` | Similar to loading global variables. The attribute's | [2]_ | Mark Shannon | +| Load | ``o.attr`` | Similar to loading global variables. The attribute's | [#load-attr]_ | Mark Shannon | | attribute | | index inside the class/object's namespace is cached. | | | | | | In most cases, attribute loading will require zero | | | | | | namespace lookups. | | | @@ -1484,14 +1492,15 @@ Bucher, with additional help from Irit Katriel and Dennis Sweeney.) | Store | ``o.attr = z`` | Similar to load attribute optimization. | 2% | Mark Shannon | | attribute | | | in pyperformance | | +---------------+--------------------+-------------------------------------------------------+-------------------+-------------------+ -| Unpack | ``*seq`` | Specialized for common containers such as ``list`` | 8% | Brandt Bucher | -| Sequence | | and ``tuple``. Avoids internal calling convention. | | | +| Unpack | ``*seq`` | Specialized for common containers such as | 8% | Brandt Bucher | +| Sequence | | :class:`list` and :class:`tuple`. | | | +| | | Avoids internal calling convention. | | | +---------------+--------------------+-------------------------------------------------------+-------------------+-------------------+ -.. [1] A similar optimization already existed since Python 3.8. 3.11 - specializes for more forms and reduces some overhead. +.. [#load-global] A similar optimization already existed since Python 3.8. + 3.11 specializes for more forms and reduces some overhead. -.. [2] A similar optimization already existed since Python 3.10. +.. [#load-attr] A similar optimization already existed since Python 3.10. 3.11 specializes for more forms. Furthermore, all attribute loads should be sped up by :issue:`45947`. @@ -1501,49 +1510,72 @@ Bucher, with additional help from Irit Katriel and Dennis Sweeney.) Misc ---- -* Objects now require less memory due to lazily created object namespaces. Their - namespace dictionaries now also share keys more freely. +* Objects now require less memory due to lazily created object namespaces. + Their namespace dictionaries now also share keys more freely. (Contributed Mark Shannon in :issue:`45340` and :issue:`40116`.) +* "Zero-cost" exceptions are implemented, eliminating the cost + of :keyword:`try` statements when no exception is raised. + (Contributed by Mark Shannon in :issue:`40222`.) + * A more concise representation of exceptions in the interpreter reduced the time required for catching an exception by about 10%. (Contributed by Irit Katriel in :issue:`45711`.) +* :mod:`re`'s regular expression matching engine has been partially refactored, + and now uses computed gotos (or "threaded code") on supported platforms. As a + result, Python 3.11 executes the `pyperformance regular expression benchmarks + `_ up to 10% + faster than Python 3.10. + (Contributed by Brandt Bucher in :gh:`91404`.) + .. _whatsnew311-faster-cpython-faq: FAQ --- -| Q: How should I write my code to utilize these speedups? -| -| A: You don't have to change your code. Write Pythonic code that follows common - best practices. The Faster CPython project optimizes for common code - patterns we observe. -| -| -| Q: Will CPython 3.11 use more memory? -| -| A: Maybe not. We don't expect memory use to exceed 20% more than 3.10. - This is offset by memory optimizations for frame objects and object - dictionaries as mentioned above. -| -| -| Q: I don't see any speedups in my workload. Why? -| -| A: Certain code won't have noticeable benefits. If your code spends most of - its time on I/O operations, or already does most of its - computation in a C extension library like numpy, there won't be significant - speedup. This project currently benefits pure-Python workloads the most. -| -| Furthermore, the pyperformance figures are a geometric mean. Even within the - pyperformance benchmarks, certain benchmarks have slowed down slightly, while - others have sped up by nearly 2x! -| -| -| Q: Is there a JIT compiler? -| -| A: No. We're still exploring other optimizations. +.. _faster-cpython-faq-my-code: + +How should I write my code to utilize these speedups? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Write Pythonic code that follows common best practices; +you don't have to change your code. +The Faster CPython project optimizes for common code patterns we observe. + + +.. _faster-cpython-faq-memory: + +Will CPython 3.11 use more memory? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Maybe not; we don't expect memory use to exceed 20% higher than 3.10. +This is offset by memory optimizations for frame objects and object +dictionaries as mentioned above. + + +.. _faster-cpython-ymmv: + +I don't see any speedups in my workload. Why? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Certain code won't have noticeable benefits. If your code spends most of +its time on I/O operations, or already does most of its +computation in a C extension library like NumPy, there won't be significant +speedups. This project currently benefits pure-Python workloads the most. + +Furthermore, the pyperformance figures are a geometric mean. Even within the +pyperformance benchmarks, certain benchmarks have slowed down slightly, while +others have sped up by nearly 2x! + + +.. _faster-cpython-jit: + +Is there a JIT compiler? +^^^^^^^^^^^^^^^^^^^^^^^^ + +No. We're still exploring other optimizations. .. _whatsnew311-faster-cpython-about: From d8485d6c8bd7ad8191fde337d81c01f2192855eb Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Tue, 7 Mar 2023 08:44:55 +0530 Subject: [PATCH 27/97] Remove redundant `_ensure_future` in favor of `ensure_future` in `asyncio` (#102398) --- Lib/asyncio/tasks.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index a2e06d5ef72f42f..1c20754b839b69d 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -630,10 +630,6 @@ def ensure_future(coro_or_future, *, loop=None): If the argument is a Future, it is returned directly. """ - return _ensure_future(coro_or_future, loop=loop) - - -def _ensure_future(coro_or_future, *, loop=None): if futures.isfuture(coro_or_future): if loop is not None and loop is not futures._get_loop(coro_or_future): raise ValueError('The future belongs to a different loop than ' @@ -798,7 +794,7 @@ def _done_callback(fut): outer = None # bpo-46672 for arg in coros_or_futures: if arg not in arg_to_fut: - fut = _ensure_future(arg, loop=loop) + fut = ensure_future(arg, loop=loop) if loop is None: loop = futures._get_loop(fut) if fut is not arg: @@ -855,7 +851,7 @@ def shield(arg): weak references to tasks. A task that isn't referenced elsewhere may get garbage collected at any time, even before it's done. """ - inner = _ensure_future(arg) + inner = ensure_future(arg) if inner.done(): # Shortcut. return inner From 4a3ea1fdd890e5e2ec26540dc3c958a52fba6556 Mon Sep 17 00:00:00 2001 From: "C.A.M. Gerlach" Date: Tue, 7 Mar 2023 08:38:31 -0600 Subject: [PATCH 28/97] gh-95913: Consolidate build requirements changes in 3.11 WhatsNew (GH-98781) Apply suggestion to combine build requirements changes in 3.11 WhatsNew Co-authored-by: Petr Viktorin --- Doc/whatsnew/3.11.rst | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 391ea531281a483..10fcfb6a0b56396 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -2114,30 +2114,22 @@ Build Changes and WASI contributed by Christian Heimes in :gh:`90473`; platforms promoted in :gh:`95085`) -* Building Python now requires: +* Building CPython now requires: - * A `C11 `_ compiler. + * A `C11 `_ compiler and standard library. `Optional C11 features `_ are not required. - (Contributed by Victor Stinner in :issue:`46656`.) + (Contributed by Victor Stinner in :issue:`46656`, + :issue:`45440` and :issue:`46640`.) * Support for `IEEE 754 `_ floating point numbers. (Contributed by Victor Stinner in :issue:`46917`.) - * Support for `floating point Not-a-Number (NaN) - `_, - as the :c:macro:`!Py_NO_NAN` macro has been removed. - (Contributed by Victor Stinner in :issue:`46656`.) - - * A `C99 `_ - ```` header file providing the - :c:func:`!copysign`, :c:func:`!hypot`, :c:func:`!isfinite`, - :c:func:`!isinf`, :c:func:`!isnan`, and :c:func:`!round` functions - (contributed by Victor Stinner in :issue:`45440`); - and a :c:data:`!NAN` constant or the :c:func:`!__builtin_nan` function - (Contributed by Victor Stinner in :issue:`46640`). +* The :c:macro:`!Py_NO_NAN` macro has been removed. + Since CPython now requires IEEE 754 floats, NaN values are always available. + (Contributed by Victor Stinner in :issue:`46656`.) * The :mod:`tkinter` package now requires `Tcl/Tk `_ version 8.5.12 or newer. From 54060ae91da2df44b3f6e6c698694d40284687e9 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Tue, 7 Mar 2023 18:16:32 +0000 Subject: [PATCH 29/97] gh-87092: compiler's CFG construction moved to after codegen stage (#102320) --- Python/compile.c | 520 ++++++++++++++++++++++++++++++----------------- 1 file changed, 336 insertions(+), 184 deletions(-) diff --git a/Python/compile.c b/Python/compile.c index 2e60a8157533ad9..45c97b4f8ef0e65 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -175,19 +175,18 @@ static struct jump_target_label_ NO_LABEL = {-1}; #define IS_LABEL(L) (!SAME_LABEL((L), (NO_LABEL))) #define NEW_JUMP_TARGET_LABEL(C, NAME) \ - jump_target_label NAME = cfg_new_label(CFG_BUILDER(C)); \ + jump_target_label NAME = instr_sequence_new_label(INSTR_SEQUENCE(C)); \ if (!IS_LABEL(NAME)) { \ return ERROR; \ } #define USE_LABEL(C, LBL) \ - RETURN_IF_ERROR(cfg_builder_use_label(CFG_BUILDER(C), LBL)) + RETURN_IF_ERROR(instr_sequence_use_label(INSTR_SEQUENCE(C), (LBL).id)) -struct instr { +struct cfg_instr { int i_opcode; int i_oparg; location i_loc; - /* The following fields should not be set by the front-end: */ struct basicblock_ *i_target; /* target block (if jump instruction) */ struct basicblock_ *i_except; /* target block when exception is raised */ }; @@ -196,7 +195,7 @@ struct instr { #define INSTR_SET_OP1(I, OP, ARG) \ do { \ assert(HAS_ARG(OP)); \ - struct instr *_instr__ptr_ = (I); \ + struct cfg_instr *_instr__ptr_ = (I); \ _instr__ptr_->i_opcode = (OP); \ _instr__ptr_->i_oparg = (ARG); \ } while (0); @@ -205,7 +204,7 @@ struct instr { #define INSTR_SET_OP0(I, OP) \ do { \ assert(!HAS_ARG(OP)); \ - struct instr *_instr__ptr_ = (I); \ + struct cfg_instr *_instr__ptr_ = (I); \ _instr__ptr_->i_opcode = (OP); \ _instr__ptr_->i_oparg = 0; \ } while (0); @@ -235,25 +234,25 @@ is_bit_set_in_table(const uint32_t *table, int bitindex) { } static inline int -is_relative_jump(struct instr *i) +is_relative_jump(struct cfg_instr *i) { return is_bit_set_in_table(_PyOpcode_RelativeJump, i->i_opcode); } static inline int -is_block_push(struct instr *i) +is_block_push(struct cfg_instr *i) { return IS_BLOCK_PUSH_OPCODE(i->i_opcode); } static inline int -is_jump(struct instr *i) +is_jump(struct cfg_instr *i) { return IS_JUMP_OPCODE(i->i_opcode); } static int -instr_size(struct instr *instruction) +instr_size(struct cfg_instr *instruction) { int opcode = instruction->i_opcode; assert(!IS_PSEUDO_OPCODE(opcode)); @@ -265,7 +264,7 @@ instr_size(struct instr *instruction) } static void -write_instr(_Py_CODEUNIT *codestr, struct instr *instruction, int ilen) +write_instr(_Py_CODEUNIT *codestr, struct cfg_instr *instruction, int ilen) { int opcode = instruction->i_opcode; assert(!IS_PSEUDO_OPCODE(opcode)); @@ -313,7 +312,7 @@ typedef struct basicblock_ { /* Exception stack at start of block, used by assembler to create the exception handling table */ ExceptStack *b_exceptstack; /* pointer to an array of instructions, initially NULL */ - struct instr *b_instr; + struct cfg_instr *b_instr; /* If b_next is non-NULL, it is a pointer to the next block reached by normal control flow. */ struct basicblock_ *b_next; @@ -342,7 +341,7 @@ typedef struct basicblock_ { } basicblock; -static struct instr * +static struct cfg_instr * basicblock_last_instr(const basicblock *b) { assert(b->b_iused >= 0); if (b->b_iused > 0) { @@ -352,21 +351,15 @@ basicblock_last_instr(const basicblock *b) { return NULL; } -static inline int -basicblock_returns(const basicblock *b) { - struct instr *last = basicblock_last_instr(b); - return last && (last->i_opcode == RETURN_VALUE || last->i_opcode == RETURN_CONST); -} - static inline int basicblock_exits_scope(const basicblock *b) { - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); return last && IS_SCOPE_EXIT_OPCODE(last->i_opcode); } static inline int basicblock_nofallthrough(const basicblock *b) { - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); return (last && (IS_SCOPE_EXIT_OPCODE(last->i_opcode) || IS_UNCONDITIONAL_JUMP_OPCODE(last->i_opcode))); @@ -415,10 +408,208 @@ typedef struct cfg_builder_ { basicblock *g_curblock; /* label for the next instruction to be placed */ jump_target_label g_current_label; - /* next free label id */ - int g_next_free_label; } cfg_builder; +typedef struct { + int i_opcode; + int i_oparg; + location i_loc; +} instruction; + + +typedef struct instr_sequence_ { + instruction *s_instrs; + int s_allocated; + int s_used; + + int *s_labelmap; /* label id --> instr offset */ + int s_labelmap_size; + int s_next_free_label; /* next free label id */ +} instr_sequence; + +#define INITIAL_INSTR_SEQUENCE_SIZE 100 +#define INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE 10 + +/* + * Resize the array if index is out of range. + * + * idx: the index we want to access + * arr: pointer to the array + * alloc: pointer to the capacity of the array + * default_alloc: initial number of items + * item_size: size of each item + * + */ +static int +ensure_array_large_enough(int idx, void **arr_, int *alloc, int default_alloc, size_t item_size) +{ + void *arr = *arr_; + if (arr == NULL) { + int new_alloc = default_alloc; + if (idx >= new_alloc) { + new_alloc = idx + default_alloc; + } + arr = PyObject_Calloc(new_alloc, item_size); + if (arr == NULL) { + PyErr_NoMemory(); + return ERROR; + } + *alloc = new_alloc; + } + else if (idx >= *alloc) { + size_t oldsize = *alloc * item_size; + int new_alloc = *alloc << 1; + if (idx >= new_alloc) { + new_alloc = idx + default_alloc; + } + size_t newsize = new_alloc * item_size; + + if (oldsize > (SIZE_MAX >> 1)) { + PyErr_NoMemory(); + return ERROR; + } + + assert(newsize > 0); + void *tmp = PyObject_Realloc(arr, newsize); + if (tmp == NULL) { + PyErr_NoMemory(); + return ERROR; + } + *alloc = new_alloc; + arr = tmp; + memset((char *)arr + oldsize, 0, newsize - oldsize); + } + + *arr_ = arr; + return SUCCESS; +} + +static int +instr_sequence_next_inst(instr_sequence *seq) { + assert(seq->s_instrs != NULL || seq->s_used == 0); + + RETURN_IF_ERROR( + ensure_array_large_enough(seq->s_used + 1, + (void**)&seq->s_instrs, + &seq->s_allocated, + INITIAL_INSTR_SEQUENCE_SIZE, + sizeof(instruction))); + assert(seq->s_used < seq->s_allocated); + return seq->s_used++; +} + +static jump_target_label +instr_sequence_new_label(instr_sequence *seq) +{ + jump_target_label lbl = {seq->s_next_free_label++}; + return lbl; +} + +static int +instr_sequence_use_label(instr_sequence *seq, int lbl) { + int old_size = seq->s_labelmap_size; + RETURN_IF_ERROR( + ensure_array_large_enough(lbl, + (void**)&seq->s_labelmap, + &seq->s_labelmap_size, + INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE, + sizeof(int))); + + for(int i = old_size; i < seq->s_labelmap_size; i++) { + seq->s_labelmap[i] = -111; /* something weird, for debugging */ + } + seq->s_labelmap[lbl] = seq->s_used; /* label refers to the next instruction */ + return SUCCESS; +} + +static int +instr_sequence_addop(instr_sequence *seq, int opcode, int oparg, location loc) +{ + assert(IS_WITHIN_OPCODE_RANGE(opcode)); + assert(!IS_ASSEMBLER_OPCODE(opcode)); + assert(HAS_ARG(opcode) || HAS_TARGET(opcode) || oparg == 0); + assert(0 <= oparg && oparg < (1 << 30)); + + int idx = instr_sequence_next_inst(seq); + RETURN_IF_ERROR(idx); + instruction *ci = &seq->s_instrs[idx]; + ci->i_opcode = opcode; + ci->i_oparg = oparg; + ci->i_loc = loc; + return SUCCESS; +} + +static int +instr_sequence_insert_instruction(instr_sequence *seq, int pos, + int opcode, int oparg, location loc) +{ + assert(pos >= 0 && pos <= seq->s_used); + int last_idx = instr_sequence_next_inst(seq); + RETURN_IF_ERROR(last_idx); + for (int i=last_idx-1; i >= pos; i--) { + seq->s_instrs[i+1] = seq->s_instrs[i]; + } + instruction *ci = &seq->s_instrs[pos]; + ci->i_opcode = opcode; + ci->i_oparg = oparg; + ci->i_loc = loc; + + /* fix the labels map */ + for(int lbl=0; lbl < seq->s_labelmap_size; lbl++) { + if (seq->s_labelmap[lbl] >= pos) { + seq->s_labelmap[lbl]++; + } + } + return SUCCESS; +} + +static void +instr_sequence_fini(instr_sequence *seq) { + PyObject_Free(seq->s_labelmap); + seq->s_labelmap = NULL; + + PyObject_Free(seq->s_instrs); + seq->s_instrs = NULL; +} + +static int basicblock_addop(basicblock *b, int opcode, int oparg, location loc); +static int cfg_builder_maybe_start_new_block(cfg_builder *g); + +static int +cfg_builder_use_label(cfg_builder *g, jump_target_label lbl) +{ + g->g_current_label = lbl; + return cfg_builder_maybe_start_new_block(g); +} + +static int +cfg_builder_addop(cfg_builder *g, int opcode, int oparg, location loc) +{ + RETURN_IF_ERROR(cfg_builder_maybe_start_new_block(g)); + return basicblock_addop(g->g_curblock, opcode, oparg, loc); +} + +static int cfg_builder_init(cfg_builder *g); + +static int +instr_sequence_to_cfg(instr_sequence *seq, cfg_builder *g) { + memset(g, 0, sizeof(cfg_builder)); + RETURN_IF_ERROR(cfg_builder_init(g)); + /* Note: there can be more than one label for the same offset */ + for (int i = 0; i < seq->s_used; i++) { + for (int j=0; j < seq->s_labelmap_size; j++) { + if (seq->s_labelmap[j] == i) { + jump_target_label lbl = {j}; + RETURN_IF_ERROR(cfg_builder_use_label(g, lbl)); + } + } + instruction *instr = &seq->s_instrs[i]; + RETURN_IF_ERROR(cfg_builder_addop(g, instr->i_opcode, instr->i_oparg, instr->i_loc)); + } + return SUCCESS; +} + + /* The following items change on entry and exit of code blocks. They must be saved and restored when returning to a block. */ @@ -445,7 +636,7 @@ struct compiler_unit { Py_ssize_t u_posonlyargcount; /* number of positional only arguments for block */ Py_ssize_t u_kwonlyargcount; /* number of keyword only arguments for block */ - cfg_builder u_cfg_builder; /* The control flow graph */ + instr_sequence u_instr_sequence; /* codegen output */ int u_nfblocks; struct fblockinfo u_fblock[CO_MAXBLOCKS]; @@ -481,7 +672,7 @@ struct compiler { PyArena *c_arena; /* pointer to memory allocation arena */ }; -#define CFG_BUILDER(C) (&((C)->u->u_cfg_builder)) +#define INSTR_SEQUENCE(C) (&((C)->u->u_instr_sequence)) typedef struct { @@ -511,9 +702,7 @@ typedef struct { static int basicblock_next_instr(basicblock *); -static basicblock *cfg_builder_new_block(cfg_builder *g); -static int cfg_builder_maybe_start_new_block(cfg_builder *g); -static int cfg_builder_addop_i(cfg_builder *g, int opcode, Py_ssize_t oparg, location loc); +static int codegen_addop_i(instr_sequence *seq, int opcode, Py_ssize_t oparg, location loc); static void compiler_free(struct compiler *); static int compiler_error(struct compiler *, location loc, const char *, ...); @@ -744,7 +933,8 @@ dictbytype(PyObject *src, int scope_type, int flag, Py_ssize_t offset) return dest; } -static void +#ifndef NDEBUG +static bool cfg_builder_check(cfg_builder *g) { for (basicblock *block = g->g_block_list; block != NULL; block = block->b_list) { @@ -759,7 +949,11 @@ cfg_builder_check(cfg_builder *g) assert (block->b_ialloc == 0); } } + return true; } +#endif + +static basicblock *cfg_builder_new_block(cfg_builder *g); static int cfg_builder_init(cfg_builder *g) @@ -777,7 +971,7 @@ cfg_builder_init(cfg_builder *g) static void cfg_builder_fini(cfg_builder* g) { - cfg_builder_check(g); + assert(cfg_builder_check(g)); basicblock *b = g->g_block_list; while (b != NULL) { if (b->b_instr) { @@ -792,7 +986,7 @@ cfg_builder_fini(cfg_builder* g) static void compiler_unit_free(struct compiler_unit *u) { - cfg_builder_fini(&u->u_cfg_builder); + instr_sequence_fini(&u->u_instr_sequence); Py_CLEAR(u->u_ste); Py_CLEAR(u->u_name); Py_CLEAR(u->u_qualname); @@ -878,13 +1072,6 @@ compiler_set_qualname(struct compiler *c) return SUCCESS; } -static jump_target_label -cfg_new_label(cfg_builder *g) -{ - jump_target_label lbl = {g->g_next_free_label++}; - return lbl; -} - /* Allocate a new block and return a pointer to it. Returns NULL on error. */ @@ -912,13 +1099,6 @@ cfg_builder_use_next_block(cfg_builder *g, basicblock *block) return block; } -static int -cfg_builder_use_label(cfg_builder *g, jump_target_label lbl) -{ - g->g_current_label = lbl; - return cfg_builder_maybe_start_new_block(g); -} - static inline int basicblock_append_instructions(basicblock *target, basicblock *source) { @@ -958,40 +1138,15 @@ static int basicblock_next_instr(basicblock *b) { assert(b != NULL); - if (b->b_instr == NULL) { - b->b_instr = (struct instr *)PyObject_Calloc( - DEFAULT_BLOCK_SIZE, sizeof(struct instr)); - if (b->b_instr == NULL) { - PyErr_NoMemory(); - return ERROR; - } - b->b_ialloc = DEFAULT_BLOCK_SIZE; - } - else if (b->b_iused == b->b_ialloc) { - struct instr *tmp; - size_t oldsize, newsize; - oldsize = b->b_ialloc * sizeof(struct instr); - newsize = oldsize << 1; - if (oldsize > (SIZE_MAX >> 1)) { - PyErr_NoMemory(); - return ERROR; - } + RETURN_IF_ERROR( + ensure_array_large_enough( + b->b_iused + 1, + (void**)&b->b_instr, + &b->b_ialloc, + DEFAULT_BLOCK_SIZE, + sizeof(struct cfg_instr))); - if (newsize == 0) { - PyErr_NoMemory(); - return ERROR; - } - b->b_ialloc <<= 1; - tmp = (struct instr *)PyObject_Realloc( - (void *)b->b_instr, newsize); - if (tmp == NULL) { - PyErr_NoMemory(); - return ERROR; - } - b->b_instr = tmp; - memset((char *)b->b_instr + oldsize, 0, newsize - oldsize); - } return b->b_iused++; } @@ -1100,7 +1255,7 @@ basicblock_addop(basicblock *b, int opcode, int oparg, location loc) if (off < 0) { return ERROR; } - struct instr *i = &b->b_instr[off]; + struct cfg_instr *i = &b->b_instr[off]; i->i_opcode = opcode; i->i_oparg = oparg; i->i_target = NULL; @@ -1115,7 +1270,7 @@ cfg_builder_current_block_is_terminated(cfg_builder *g) if (IS_LABEL(g->g_current_label)) { return true; } - struct instr *last = basicblock_last_instr(g->g_curblock); + struct cfg_instr *last = basicblock_last_instr(g->g_curblock); return last && IS_TERMINATOR_OPCODE(last->i_opcode); } @@ -1135,17 +1290,10 @@ cfg_builder_maybe_start_new_block(cfg_builder *g) } static int -cfg_builder_addop(cfg_builder *g, int opcode, int oparg, location loc) -{ - RETURN_IF_ERROR(cfg_builder_maybe_start_new_block(g)); - return basicblock_addop(g->g_curblock, opcode, oparg, loc); -} - -static int -cfg_builder_addop_noarg(cfg_builder *g, int opcode, location loc) +codegen_addop_noarg(instr_sequence *seq, int opcode, location loc) { assert(!HAS_ARG(opcode)); - return cfg_builder_addop(g, opcode, 0, loc); + return instr_sequence_addop(seq, opcode, 0, loc); } static Py_ssize_t @@ -1303,7 +1451,7 @@ compiler_addop_load_const(struct compiler *c, location loc, PyObject *o) if (arg < 0) { return ERROR; } - return cfg_builder_addop_i(CFG_BUILDER(c), LOAD_CONST, arg, loc); + return codegen_addop_i(INSTR_SEQUENCE(c), LOAD_CONST, arg, loc); } static int @@ -1314,7 +1462,7 @@ compiler_addop_o(struct compiler *c, location loc, if (arg < 0) { return ERROR; } - return cfg_builder_addop_i(CFG_BUILDER(c), opcode, arg, loc); + return codegen_addop_i(INSTR_SEQUENCE(c), opcode, arg, loc); } static int @@ -1340,12 +1488,12 @@ compiler_addop_name(struct compiler *c, location loc, arg <<= 1; arg |= 1; } - return cfg_builder_addop_i(CFG_BUILDER(c), opcode, arg, loc); + return codegen_addop_i(INSTR_SEQUENCE(c), opcode, arg, loc); } /* Add an opcode with an integer argument */ static int -cfg_builder_addop_i(cfg_builder *g, int opcode, Py_ssize_t oparg, location loc) +codegen_addop_i(instr_sequence *seq, int opcode, Py_ssize_t oparg, location loc) { /* oparg value is unsigned, but a signed C int is usually used to store it in the C code (like Python/ceval.c). @@ -1356,23 +1504,23 @@ cfg_builder_addop_i(cfg_builder *g, int opcode, Py_ssize_t oparg, location loc) EXTENDED_ARG is used for 16, 24, and 32-bit arguments. */ int oparg_ = Py_SAFE_DOWNCAST(oparg, Py_ssize_t, int); - return cfg_builder_addop(g, opcode, oparg_, loc); + return instr_sequence_addop(seq, opcode, oparg_, loc); } static int -cfg_builder_addop_j(cfg_builder *g, location loc, +codegen_addop_j(instr_sequence *seq, location loc, int opcode, jump_target_label target) { assert(IS_LABEL(target)); assert(IS_JUMP_OPCODE(opcode) || IS_BLOCK_PUSH_OPCODE(opcode)); - return cfg_builder_addop(g, opcode, target.id, loc); + return instr_sequence_addop(seq, opcode, target.id, loc); } #define ADDOP(C, LOC, OP) \ - RETURN_IF_ERROR(cfg_builder_addop_noarg(CFG_BUILDER(C), (OP), (LOC))) + RETURN_IF_ERROR(codegen_addop_noarg(INSTR_SEQUENCE(C), (OP), (LOC))) #define ADDOP_IN_SCOPE(C, LOC, OP) { \ - if (cfg_builder_addop_noarg(CFG_BUILDER(C), (OP), (LOC)) < 0) { \ + if (codegen_addop_noarg(INSTR_SEQUENCE(C), (OP), (LOC)) < 0) { \ compiler_exit_scope(C); \ return ERROR; \ } \ @@ -1407,10 +1555,10 @@ cfg_builder_addop_j(cfg_builder *g, location loc, RETURN_IF_ERROR(compiler_addop_name((C), (LOC), (OP), (C)->u->u_ ## TYPE, (O))) #define ADDOP_I(C, LOC, OP, O) \ - RETURN_IF_ERROR(cfg_builder_addop_i(CFG_BUILDER(C), (OP), (O), (LOC))) + RETURN_IF_ERROR(codegen_addop_i(INSTR_SEQUENCE(C), (OP), (O), (LOC))) #define ADDOP_JUMP(C, LOC, OP, O) \ - RETURN_IF_ERROR(cfg_builder_addop_j(CFG_BUILDER(C), (LOC), (OP), (O))) + RETURN_IF_ERROR(codegen_addop_j(INSTR_SEQUENCE(C), (LOC), (OP), (O))) #define ADDOP_COMPARE(C, LOC, CMP) \ RETURN_IF_ERROR(compiler_addcompare((C), (LOC), (cmpop_ty)(CMP))) @@ -1546,9 +1694,6 @@ compiler_enter_scope(struct compiler *c, identifier name, c->c_nestlevel++; - cfg_builder *g = CFG_BUILDER(c); - RETURN_IF_ERROR(cfg_builder_init(g)); - if (u->u_scope_type == COMPILER_SCOPE_MODULE) { loc.lineno = 0; } @@ -1582,7 +1727,6 @@ compiler_exit_scope(struct compiler *c) _PyErr_WriteUnraisableMsg("on removing the last compiler " "stack item", NULL); } - cfg_builder_check(CFG_BUILDER(c)); } else { c->u = NULL; @@ -2321,7 +2465,7 @@ compiler_check_debug_args(struct compiler *c, arguments_ty args) } static inline int -insert_instruction(basicblock *block, int pos, struct instr *instr) { +insert_instruction(basicblock *block, int pos, struct cfg_instr *instr) { RETURN_IF_ERROR(basicblock_next_instr(block)); for (int i = block->b_iused - 1; i > pos; i--) { block->b_instr[i] = block->b_instr[i-1]; @@ -2336,14 +2480,10 @@ wrap_in_stopiteration_handler(struct compiler *c) NEW_JUMP_TARGET_LABEL(c, handler); /* Insert SETUP_CLEANUP at start */ - struct instr setup = { - .i_opcode = SETUP_CLEANUP, - .i_oparg = handler.id, - .i_loc = NO_LOCATION, - .i_target = NULL, - }; RETURN_IF_ERROR( - insert_instruction(c->u->u_cfg_builder.g_entryblock, 0, &setup)); + instr_sequence_insert_instruction( + INSTR_SEQUENCE(c), 0, + SETUP_CLEANUP, handler.id, NO_LOCATION)); ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); ADDOP(c, NO_LOCATION, RETURN_VALUE); @@ -4032,7 +4172,7 @@ compiler_nameop(struct compiler *c, location loc, if (op == LOAD_GLOBAL) { arg <<= 1; } - return cfg_builder_addop_i(CFG_BUILDER(c), op, arg, loc); + return codegen_addop_i(INSTR_SEQUENCE(c), op, arg, loc); } static int @@ -6051,7 +6191,7 @@ emit_and_reset_fail_pop(struct compiler *c, location loc, } while (--pc->fail_pop_size) { USE_LABEL(c, pc->fail_pop[pc->fail_pop_size]); - if (cfg_builder_addop_noarg(CFG_BUILDER(c), POP_TOP, loc) < 0) { + if (codegen_addop_noarg(INSTR_SEQUENCE(c), POP_TOP, loc) < 0) { pc->fail_pop_size = 0; PyObject_Free(pc->fail_pop); pc->fail_pop = NULL; @@ -6491,7 +6631,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) pc->fail_pop = NULL; pc->fail_pop_size = 0; pc->on_top = 0; - if (cfg_builder_addop_i(CFG_BUILDER(c), COPY, 1, LOC(alt)) < 0 || + if (codegen_addop_i(INSTR_SEQUENCE(c), COPY, 1, LOC(alt)) < 0 || compiler_pattern(c, alt, pc) < 0) { goto error; } @@ -6554,7 +6694,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) } } assert(control); - if (cfg_builder_addop_j(CFG_BUILDER(c), LOC(alt), JUMP, end) < 0 || + if (codegen_addop_j(INSTR_SEQUENCE(c), LOC(alt), JUMP, end) < 0 || emit_and_reset_fail_pop(c, LOC(alt), pc) < 0) { goto error; @@ -6566,7 +6706,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) // Need to NULL this for the PyObject_Free call in the error block. old_pc.fail_pop = NULL; // No match. Pop the remaining copy of the subject and fail: - if (cfg_builder_addop_noarg(CFG_BUILDER(c), POP_TOP, LOC(p)) < 0 || + if (codegen_addop_noarg(INSTR_SEQUENCE(c), POP_TOP, LOC(p)) < 0 || jump_to_fail_pop(c, LOC(p), pc, JUMP) < 0) { goto error; } @@ -6884,7 +7024,7 @@ stackdepth(basicblock *entryblock, int code_flags) assert(depth >= 0); basicblock *next = b->b_next; for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; int effect = stack_effect(instr->i_opcode, instr->i_oparg, 0); if (effect == PY_INVALID_STACK_EFFECT) { PyErr_Format(PyExc_SystemError, @@ -6973,7 +7113,7 @@ blocksize(basicblock *b) } static basicblock * -push_except_block(ExceptStack *stack, struct instr *setup) { +push_except_block(ExceptStack *stack, struct cfg_instr *setup) { assert(is_block_push(setup)); int opcode = setup->i_opcode; basicblock * target = setup->i_target; @@ -7045,7 +7185,7 @@ label_exception_targets(basicblock *entryblock) { b->b_exceptstack = NULL; handler = except_stack_top(except_stack); for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; if (is_block_push(instr)) { if (!instr->i_target->b_visited) { ExceptStack *copy = copy_except_stack(except_stack); @@ -7123,7 +7263,7 @@ mark_except_handlers(basicblock *entryblock) { #endif for (basicblock *b = entryblock; b != NULL; b = b->b_next) { for (int i=0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; if (is_block_push(instr)) { instr->i_target->b_except_handler = 1; } @@ -7152,7 +7292,7 @@ mark_warm(basicblock *entryblock) { next->b_visited = 1; } for (int i=0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; if (is_jump(instr) && !instr->i_target->b_visited) { *sp++ = instr->i_target; instr->i_target->b_visited = 1; @@ -7197,7 +7337,7 @@ mark_cold(basicblock *entryblock) { } } for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; if (is_jump(instr)) { assert(i == b->b_iused - 1); basicblock *target = b->b_instr[i].i_target; @@ -7238,7 +7378,7 @@ push_cold_blocks_to_end(cfg_builder *g, int code_flags) { b->b_next = explicit_jump; /* set target */ - struct instr *last = basicblock_last_instr(explicit_jump); + struct cfg_instr *last = basicblock_last_instr(explicit_jump); last->i_target = explicit_jump->b_next; } } @@ -7293,7 +7433,7 @@ static void convert_exception_handlers_to_nops(basicblock *entryblock) { for (basicblock *b = entryblock; b != NULL; b = b->b_next) { for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; if (is_block_push(instr) || instr->i_opcode == POP_BLOCK) { INSTR_SET_OP0(instr, NOP); } @@ -7372,7 +7512,7 @@ assemble_exception_table(struct assembler *a, basicblock *entryblock) for (b = entryblock; b != NULL; b = b->b_next) { ioffset = b->b_offset; for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; if (instr->i_except != handler) { if (handler != NULL) { RETURN_IF_ERROR( @@ -7542,7 +7682,7 @@ assemble_emit_location(struct assembler* a, location loc, int isize) */ static int -assemble_emit(struct assembler *a, struct instr *i) +assemble_emit(struct assembler *a, struct cfg_instr *i) { Py_ssize_t len = PyBytes_GET_SIZE(a->a_bytecode); _Py_CODEUNIT *code; @@ -7562,7 +7702,7 @@ assemble_emit(struct assembler *a, struct instr *i) static int normalize_jumps_in_block(cfg_builder *g, basicblock *b) { - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); if (last == NULL || !is_jump(last)) { return SUCCESS; } @@ -7663,7 +7803,7 @@ assemble_jump_offsets(basicblock *entryblock) for (basicblock *b = entryblock; b != NULL; b = b->b_next) { bsize = b->b_offset; for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; int isize = instr_size(instr); /* Relative jumps are computed relative to the instruction pointer after fetching @@ -7735,7 +7875,7 @@ scan_block_for_locals(basicblock *b, basicblock ***sp) // bit i is set if local i is potentially uninitialized uint64_t unsafe_mask = b->b_unsafe_locals_mask; for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; assert(instr->i_opcode != EXTENDED_ARG); assert(!IS_SUPERINSTRUCTION_OPCODE(instr->i_opcode)); if (instr->i_except != NULL) { @@ -7768,7 +7908,7 @@ scan_block_for_locals(basicblock *b, basicblock ***sp) if (b->b_next && BB_HAS_FALLTHROUGH(b)) { maybe_push(b->b_next, unsafe_mask, sp); } - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); if (last && is_jump(last)) { assert(last->i_target != NULL); maybe_push(last->i_target, unsafe_mask, sp); @@ -7791,7 +7931,7 @@ fast_scan_many_locals(basicblock *entryblock, int nlocals) for (basicblock *b = entryblock; b != NULL; b = b->b_next) { blocknum++; for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; assert(instr->i_opcode != EXTENDED_ARG); assert(!IS_SUPERINSTRUCTION_OPCODE(instr->i_opcode)); int arg = instr->i_oparg; @@ -8035,7 +8175,6 @@ makecode(struct compiler *c, struct assembler *a, PyObject *constslist, PyObject *consts = NULL; PyObject *localsplusnames = NULL; PyObject *localspluskinds = NULL; - names = dict_keys_inorder(c->u->u_names, 0); if (!names) { goto error; @@ -8121,7 +8260,7 @@ makecode(struct compiler *c, struct assembler *a, PyObject *constslist, /* For debugging purposes only */ #if 0 static void -dump_instr(struct instr *i) +dump_instr(struct cfg_instr *i) { const char *jrel = (is_relative_jump(i)) ? "jrel " : ""; const char *jabs = (is_jump(i) && !is_relative_jump(i))? "jabs " : ""; @@ -8139,6 +8278,12 @@ dump_instr(struct instr *i) i->i_loc.lineno, i->i_opcode, arg, jabs, jrel); } +static inline int +basicblock_returns(const basicblock *b) { + struct cfg_instr *last = basicblock_last_instr(b); + return last && (last->i_opcode == RETURN_VALUE || last->i_opcode == RETURN_CONST); +} + static void dump_basicblock(const basicblock *b) { @@ -8211,14 +8356,14 @@ insert_prefix_instructions(struct compiler *c, basicblock *entryblock, /* Add the generator prefix instructions. */ if (code_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) { - struct instr make_gen = { + struct cfg_instr make_gen = { .i_opcode = RETURN_GENERATOR, .i_oparg = 0, .i_loc = LOCATION(c->u->u_firstlineno, c->u->u_firstlineno, -1, -1), .i_target = NULL, }; RETURN_IF_ERROR(insert_instruction(entryblock, 0, &make_gen)); - struct instr pop_top = { + struct cfg_instr pop_top = { .i_opcode = POP_TOP, .i_oparg = 0, .i_loc = NO_LOCATION, @@ -8247,7 +8392,7 @@ insert_prefix_instructions(struct compiler *c, basicblock *entryblock, if (oldindex == -1) { continue; } - struct instr make_cell = { + struct cfg_instr make_cell = { .i_opcode = MAKE_CELL, // This will get fixed in offset_derefs(). .i_oparg = oldindex, @@ -8261,7 +8406,7 @@ insert_prefix_instructions(struct compiler *c, basicblock *entryblock, } if (nfreevars) { - struct instr copy_frees = { + struct cfg_instr copy_frees = { .i_opcode = COPY_FREE_VARS, .i_oparg = nfreevars, .i_loc = NO_LOCATION, @@ -8282,7 +8427,7 @@ guarantee_lineno_for_exits(basicblock *entryblock, int firstlineno) { int lineno = firstlineno; assert(lineno > 0); for (basicblock *b = entryblock; b != NULL; b = b->b_next) { - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); if (last == NULL) { continue; } @@ -8324,7 +8469,7 @@ fix_cell_offsets(struct compiler *c, basicblock *entryblock, int *fixedmap) // Then update offsets, either relative to locals or by cell2arg. for (basicblock *b = entryblock; b != NULL; b = b->b_next) { for (int i = 0; i < b->b_iused; i++) { - struct instr *inst = &b->b_instr[i]; + struct cfg_instr *inst = &b->b_instr[i]; // This is called before extended args are generated. assert(inst->i_opcode != EXTENDED_ARG); int oldoffset = inst->i_oparg; @@ -8364,7 +8509,7 @@ no_redundant_nops(cfg_builder *g) { static bool no_redundant_jumps(cfg_builder *g) { for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); if (last != NULL) { if (IS_UNCONDITIONAL_JUMP_OPCODE(last->i_opcode)) { assert(last->i_target != b->b_next); @@ -8382,7 +8527,7 @@ opcode_metadata_is_sane(cfg_builder *g) { bool result = true; for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; int opcode = instr->i_opcode; int oparg = instr->i_oparg; assert(opcode <= MAX_REAL_OPCODE); @@ -8426,7 +8571,7 @@ remove_redundant_jumps(cfg_builder *g) { */ assert(no_empty_basic_blocks(g)); for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); assert(last != NULL); assert(!IS_ASSEMBLER_OPCODE(last->i_opcode)); if (IS_UNCONDITIONAL_JUMP_OPCODE(last->i_opcode)) { @@ -8444,7 +8589,7 @@ remove_redundant_jumps(cfg_builder *g) { } static int -prepare_localsplus(struct compiler* c, int code_flags) +prepare_localsplus(struct compiler* c, cfg_builder *g, int code_flags) { assert(PyDict_GET_SIZE(c->u->u_varnames) < INT_MAX); assert(PyDict_GET_SIZE(c->u->u_cellvars) < INT_MAX); @@ -8460,7 +8605,6 @@ prepare_localsplus(struct compiler* c, int code_flags) return ERROR; } - cfg_builder* g = CFG_BUILDER(c); // This must be called before fix_cell_offsets(). if (insert_prefix_instructions(c, g->g_entryblock, cellfixedoffsets, nfreevars, code_flags)) { @@ -8474,20 +8618,21 @@ prepare_localsplus(struct compiler* c, int code_flags) if (numdropped < 0) { return ERROR; } + nlocalsplus -= numdropped; return nlocalsplus; } static int -add_return_at_end_of_block(struct compiler *c, int addNone) +add_return_at_end(struct compiler *c, int addNone) { - /* Make sure every block that falls off the end returns None. */ - if (!basicblock_returns(CFG_BUILDER(c)->g_curblock)) { - if (addNone) { - ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); - } - ADDOP(c, NO_LOCATION, RETURN_VALUE); + /* Make sure every instruction stream that falls off the end returns None. + * This also ensures that no jump target offsets are out of bounds. + */ + if (addNone) { + ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); } + ADDOP(c, NO_LOCATION, RETURN_VALUE); return SUCCESS; } @@ -8496,6 +8641,8 @@ assemble(struct compiler *c, int addNone) { PyCodeObject *co = NULL; PyObject *consts = NULL; + cfg_builder g_; + cfg_builder *g = &g_; struct assembler a; memset(&a, 0, sizeof(struct assembler)); @@ -8504,12 +8651,18 @@ assemble(struct compiler *c, int addNone) return NULL; } - if (add_return_at_end_of_block(c, addNone) < 0) { + if (add_return_at_end(c, addNone) < 0) { return NULL; } + /** Preprocessing **/ + if (instr_sequence_to_cfg(INSTR_SEQUENCE(c), g) < 0) { + goto error; + } + assert(cfg_builder_check(g)); + int nblocks = 0; - for (basicblock *b = CFG_BUILDER(c)->g_block_list; b != NULL; b = b->b_list) { + for (basicblock *b = g->g_block_list; b != NULL; b = b->b_list) { nblocks++; } if ((size_t)nblocks > SIZE_MAX / sizeof(basicblock *)) { @@ -8517,9 +8670,6 @@ assemble(struct compiler *c, int addNone) goto error; } - cfg_builder *g = CFG_BUILDER(c); - assert(g->g_entryblock != NULL); - /* Set firstlineno if it wasn't explicitly set. */ if (!c->u->u_firstlineno) { if (g->g_entryblock->b_instr && g->g_entryblock->b_instr->i_loc.lineno) { @@ -8530,9 +8680,8 @@ assemble(struct compiler *c, int addNone) } } - /** Preprocessing **/ /* Map labels to targets and mark exception handlers */ - if (translate_jump_labels_to_targets(g->g_entryblock)) { + if (translate_jump_labels_to_targets(g->g_entryblock) < 0) { goto error; } if (mark_except_handlers(g->g_entryblock) < 0) { @@ -8550,10 +8699,10 @@ assemble(struct compiler *c, int addNone) if (optimize_cfg(g, consts, c->c_const_cache)) { goto error; } - if (add_checks_for_loads_of_uninitialized_variables(g->g_entryblock, c) < 0) { + if (remove_unused_consts(g->g_entryblock, consts) < 0) { goto error; } - if (remove_unused_consts(g->g_entryblock, consts) < 0) { + if (add_checks_for_loads_of_uninitialized_variables(g->g_entryblock, c) < 0) { goto error; } @@ -8570,7 +8719,7 @@ assemble(struct compiler *c, int addNone) /** Assembly **/ - int nlocalsplus = prepare_localsplus(c, code_flags); + int nlocalsplus = prepare_localsplus(c, g, code_flags); if (nlocalsplus < 0) { goto error; } @@ -8587,7 +8736,6 @@ assemble(struct compiler *c, int addNone) if (normalize_jumps(g) < 0) { goto error; } - assert(no_redundant_jumps(g)); assert(opcode_metadata_is_sane(g)); @@ -8655,6 +8803,7 @@ assemble(struct compiler *c, int addNone) co = makecode(c, &a, consts, maxdepth, nlocalsplus, code_flags); error: Py_XDECREF(consts); + cfg_builder_fini(g); assemble_free(&a); return co; } @@ -8684,7 +8833,7 @@ get_const_value(int opcode, int oparg, PyObject *co_consts) */ static int fold_tuple_on_constants(PyObject *const_cache, - struct instr *inst, + struct cfg_instr *inst, int n, PyObject *consts) { /* Pre-conditions */ @@ -8753,7 +8902,7 @@ swaptimize(basicblock *block, int *ix) // NOTE: "./python -m test test_patma" serves as a good, quick stress test // for this function. Make sure to blow away cached *.pyc files first! assert(*ix < block->b_iused); - struct instr *instructions = &block->b_instr[*ix]; + struct cfg_instr *instructions = &block->b_instr[*ix]; // Find the length of the current sequence of SWAPs and NOPs, and record the // maximum depth of the stack manipulations: assert(instructions[0].i_opcode == SWAP); @@ -8854,7 +9003,7 @@ static int next_swappable_instruction(basicblock *block, int i, int lineno) { while (++i < block->b_iused) { - struct instr *instruction = &block->b_instr[i]; + struct cfg_instr *instruction = &block->b_instr[i]; if (0 <= lineno && instruction->i_loc.lineno != lineno) { // Optimizing across this instruction could cause user-visible // changes in the names bound between line tracing events! @@ -8880,7 +9029,7 @@ apply_static_swaps(basicblock *block, int i) // SWAPs are to our left, and potential swaperands are to our right: for (; 0 <= i; i--) { assert(i < block->b_iused); - struct instr *swap = &block->b_instr[i]; + struct cfg_instr *swap = &block->b_instr[i]; if (swap->i_opcode != SWAP) { if (swap->i_opcode == NOP || SWAPPABLE(swap->i_opcode)) { // Nope, but we know how to handle these. Keep looking: @@ -8903,7 +9052,7 @@ apply_static_swaps(basicblock *block, int i) } // Success! INSTR_SET_OP0(swap, NOP); - struct instr temp = block->b_instr[j]; + struct cfg_instr temp = block->b_instr[j]; block->b_instr[j] = block->b_instr[k]; block->b_instr[k] = temp; } @@ -8913,7 +9062,7 @@ apply_static_swaps(basicblock *block, int i) // target->i_target using the provided opcode. Return whether or not the // optimization was successful. static bool -jump_thread(struct instr *inst, struct instr *target, int opcode) +jump_thread(struct cfg_instr *inst, struct cfg_instr *target, int opcode) { assert(is_jump(inst)); assert(is_jump(target)); @@ -8938,11 +9087,11 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) { assert(PyDict_CheckExact(const_cache)); assert(PyList_CheckExact(consts)); - struct instr nop; + struct cfg_instr nop; INSTR_SET_OP0(&nop, NOP); - struct instr *target; + struct cfg_instr *target; for (int i = 0; i < bb->b_iused; i++) { - struct instr *inst = &bb->b_instr[i]; + struct cfg_instr *inst = &bb->b_instr[i]; int oparg = inst->i_oparg; int nextop = i+1 < bb->b_iused ? bb->b_instr[i+1].i_opcode : 0; if (HAS_TARGET(inst->i_opcode)) { @@ -9179,7 +9328,7 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) */ static int inline_small_exit_blocks(basicblock *bb) { - struct instr *last = basicblock_last_instr(bb); + struct cfg_instr *last = basicblock_last_instr(bb); if (last == NULL) { return 0; } @@ -9287,7 +9436,7 @@ mark_reachable(basicblock *entryblock) { } for (int i = 0; i < b->b_iused; i++) { basicblock *target; - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; if (is_jump(instr) || is_block_push(instr)) { target = instr->i_target; if (!target->b_visited) { @@ -9318,7 +9467,7 @@ eliminate_empty_basic_blocks(cfg_builder *g) { for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { assert(b->b_iused > 0); for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; if (HAS_TARGET(instr->i_opcode)) { basicblock *target = instr->i_target; while (target->b_iused == 0) { @@ -9342,7 +9491,7 @@ eliminate_empty_basic_blocks(cfg_builder *g) { static void propagate_line_numbers(basicblock *entryblock) { for (basicblock *b = entryblock; b != NULL; b = b->b_next) { - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); if (last == NULL) { continue; } @@ -9398,7 +9547,7 @@ translate_jump_labels_to_targets(basicblock *entryblock) } for (basicblock *b = entryblock; b != NULL; b = b->b_next) { for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; assert(instr->i_target == NULL); if (HAS_TARGET(instr->i_opcode)) { int lbl = instr->i_oparg; @@ -9584,7 +9733,7 @@ duplicate_exits_without_lineno(cfg_builder *g) */ basicblock *entryblock = g->g_entryblock; for (basicblock *b = entryblock; b != NULL; b = b->b_next) { - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); assert(last != NULL); if (is_jump(last)) { basicblock *target = last->i_target; @@ -9608,7 +9757,7 @@ duplicate_exits_without_lineno(cfg_builder *g) for (basicblock *b = entryblock; b != NULL; b = b->b_next) { if (BB_HAS_FALLTHROUGH(b) && b->b_next && b->b_iused > 0) { if (is_exit_without_lineno(b->b_next)) { - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); assert(last != NULL); b->b_next->b_instr[0].i_loc = last->i_loc; } @@ -9724,7 +9873,7 @@ cfg_to_instructions(cfg_builder *g) } for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; location loc = instr->i_loc; int arg = HAS_TARGET(instr->i_opcode) ? instr->i_target->b_label : instr->i_oparg; @@ -9783,16 +9932,19 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, goto finally; } - cfg_builder *g = CFG_BUILDER(c); - - if (translate_jump_labels_to_targets(g->g_entryblock) < 0) { + cfg_builder g; + if (instr_sequence_to_cfg(INSTR_SEQUENCE(c), &g) < 0) { + goto finally; + } + if (translate_jump_labels_to_targets(g.g_entryblock) < 0) { goto finally; } - res = cfg_to_instructions(g); + res = cfg_to_instructions(&g); finally: compiler_exit_scope(c); + cfg_builder_fini(&g); compiler_free(c); _PyArena_Free(arena); return res; @@ -9815,7 +9967,7 @@ _PyCompile_OptimizeCfg(PyObject *instructions, PyObject *consts) if (const_cache == NULL) { goto error; } - if (translate_jump_labels_to_targets(g.g_entryblock)) { + if (translate_jump_labels_to_targets(g.g_entryblock) < 0) { goto error; } if (optimize_cfg(&g, consts, const_cache) < 0) { From a33ca2ad1fcf857817cba505a788e15cf9d6ed0c Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Tue, 7 Mar 2023 21:27:46 +0000 Subject: [PATCH 30/97] gh-102493: fix normalization in PyErr_SetObject (#102502) Co-authored-by: Jelle Zijlstra --- Lib/test/test_capi/test_exceptions.py | 28 +++++++++++++++++++ ...-03-07-16-56-28.gh-issue-102493.gTXrcD.rst | 1 + Modules/_testcapi/exceptions.c | 15 ++++++++++ Python/errors.c | 16 ++++++++--- 4 files changed, 56 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-07-16-56-28.gh-issue-102493.gTXrcD.rst diff --git a/Lib/test/test_capi/test_exceptions.py b/Lib/test/test_capi/test_exceptions.py index b543a1a565a56f5..55f131699a25670 100644 --- a/Lib/test/test_capi/test_exceptions.py +++ b/Lib/test/test_capi/test_exceptions.py @@ -140,6 +140,34 @@ def test_err_restore(self): self.assertEqual(1, v.args[0]) self.assertIs(tb, v.__traceback__.tb_next) + def test_set_object(self): + + # new exception as obj is not an exception + with self.assertRaises(ValueError) as e: + _testcapi.exc_set_object(ValueError, 42) + self.assertEqual(e.exception.args, (42,)) + + # wraps the exception because unrelated types + with self.assertRaises(ValueError) as e: + _testcapi.exc_set_object(ValueError, TypeError(1,2,3)) + wrapped = e.exception.args[0] + self.assertIsInstance(wrapped, TypeError) + self.assertEqual(wrapped.args, (1, 2, 3)) + + # is superclass, so does not wrap + with self.assertRaises(PermissionError) as e: + _testcapi.exc_set_object(OSError, PermissionError(24)) + self.assertEqual(e.exception.args, (24,)) + + class Meta(type): + def __subclasscheck__(cls, sub): + 1/0 + + class Broken(Exception, metaclass=Meta): + pass + + with self.assertRaises(ZeroDivisionError) as e: + _testcapi.exc_set_object(Broken, Broken()) if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-07-16-56-28.gh-issue-102493.gTXrcD.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-07-16-56-28.gh-issue-102493.gTXrcD.rst new file mode 100644 index 000000000000000..4c4e88ca4e7c3cd --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-07-16-56-28.gh-issue-102493.gTXrcD.rst @@ -0,0 +1 @@ +Fix regression in semantics of normalisation in ``PyErr_SetObject``. diff --git a/Modules/_testcapi/exceptions.c b/Modules/_testcapi/exceptions.c index 43b88ccf261d989..a0575213987ffcf 100644 --- a/Modules/_testcapi/exceptions.c +++ b/Modules/_testcapi/exceptions.c @@ -78,6 +78,20 @@ make_exception_with_doc(PyObject *self, PyObject *args, PyObject *kwargs) return PyErr_NewExceptionWithDoc(name, doc, base, dict); } +static PyObject * +exc_set_object(PyObject *self, PyObject *args) +{ + PyObject *exc; + PyObject *obj; + + if (!PyArg_ParseTuple(args, "OO:exc_set_object", &exc, &obj)) { + return NULL; + } + + PyErr_SetObject(exc, obj); + return NULL; +} + static PyObject * raise_exception(PyObject *self, PyObject *args) { @@ -247,6 +261,7 @@ static PyMethodDef test_methods[] = { PyDoc_STR("fatal_error(message, release_gil=False): call Py_FatalError(message)")}, {"make_exception_with_doc", _PyCFunction_CAST(make_exception_with_doc), METH_VARARGS | METH_KEYWORDS}, + {"exc_set_object", exc_set_object, METH_VARARGS}, {"raise_exception", raise_exception, METH_VARARGS}, {"raise_memoryerror", raise_memoryerror, METH_NOARGS}, {"set_exc_info", test_set_exc_info, METH_VARARGS}, diff --git a/Python/errors.c b/Python/errors.c index f573bed3d63ef04..bbf6d397ce8097c 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -149,9 +149,16 @@ _PyErr_SetObject(PyThreadState *tstate, PyObject *exception, PyObject *value) exception); return; } - Py_XINCREF(value); /* Normalize the exception */ - if (value == NULL || (PyObject *)Py_TYPE(value) != exception) { + int is_subclass = 0; + if (value != NULL && PyExceptionInstance_Check(value)) { + is_subclass = PyObject_IsSubclass((PyObject *)Py_TYPE(value), exception); + if (is_subclass < 0) { + return; + } + } + Py_XINCREF(value); + if (!is_subclass) { /* We must normalize the value right now */ PyObject *fixed_value; @@ -206,9 +213,10 @@ _PyErr_SetObject(PyThreadState *tstate, PyObject *exception, PyObject *value) Py_DECREF(exc_value); } } - if (value != NULL && PyExceptionInstance_Check(value)) + assert(value != NULL); + if (PyExceptionInstance_Check(value)) tb = PyException_GetTraceback(value); - _PyErr_Restore(tstate, Py_XNewRef(exception), value, tb); + _PyErr_Restore(tstate, Py_NewRef(Py_TYPE(value)), value, tb); } void From f9774e57d84162ff0cba0b17a3dcdb93dfbce45e Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Tue, 7 Mar 2023 22:41:50 +0100 Subject: [PATCH 31/97] Python 3.12.0a6 --- Include/patchlevel.h | 4 +- Lib/pydoc_data/topics.py | 95 +- Misc/NEWS.d/3.12.0a6.rst | 821 ++++++++++++++++++ ...2-09-14-10-38-15.gh-issue-96821.Zk2a9c.rst | 3 - ...-12-18-08-33-28.gh-issue-100221.K94Ct3.rst | 2 - ...2-12-20-01-06-17.gh-issue-99942.lbmzYj.rst | 2 - ...3-01-12-00-49-16.gh-issue-99942.DUR8b4.rst | 2 - ...2-04-21-17-25-22.gh-issue-91744.FgvaMi.rst | 3 - ...-02-06-16-14-30.gh-issue-101578.PW5fA9.rst | 10 - ...3-02-09-10-38-20.gh-issue-99293.mFqfpp.rst | 2 - ...-02-14-15-53-01.gh-issue-101907.HgF1N2.rst | 1 - ...2-11-02-20-23-47.gh-issue-98627.VJkdRM.rst | 5 - ...-01-04-12-49-33.gh-issue-100719.uRPccL.rst | 3 - ...-02-07-14-56-43.gh-issue-101632.Fd1yxk.rst | 1 - ...-02-08-17-13-31.gh-issue-101696.seJhTt.rst | 1 - ...-02-10-01-15-57.gh-issue-101430.T3Gegb.rst | 2 - ...-02-10-07-21-47.gh-issue-101765.MO5LlC.rst | 1 - ...3-02-10-15-54-57.gh-issue-87849.IUVvPz.rst | 3 - ...3-02-11-23-14-06.gh-issue-84783._P5sMa.rst | 1 - ...-02-12-22-40-22.gh-issue-101857._bribG.rst | 1 - ...-02-13-18-21-14.gh-issue-101799.wpHbCn.rst | 2 - ...3-02-13-22-21-58.gh-issue-74895.esMNtq.rst | 5 - ...-02-16-16-57-23.gh-issue-101952.Zo1dlq.rst | 1 - ...-02-16-23-19-01.gh-issue-101967.Kqr1dz.rst | 1 - ...-02-17-10-12-13.gh-issue-100982.mJGJQw.rst | 2 - ...-02-20-15-18-33.gh-issue-102056.uHKuwH.rst | 1 - ...-02-22-15-15-32.gh-issue-102027.Km4G-d.rst | 2 - ...-02-24-17-59-39.gh-issue-102126.HTT8Vc.rst | 1 - ...-02-26-23-10-32.gh-issue-102250.7MUKoC.rst | 1 - ...-02-28-21-17-03.gh-issue-102336.-wL3Tm.rst | 1 - ...-03-04-20-56-12.gh-issue-102356.07KvUd.rst | 2 - ...-03-06-13-05-33.gh-issue-102416.dz6K5f.rst | 1 - ...-03-07-16-56-28.gh-issue-102493.gTXrcD.rst | 1 - ...3-02-07-21-43-24.gh-issue-97725.cuY7Cd.rst | 2 - ...3-02-19-10-33-01.gh-issue-85417.kYO8u3.rst | 1 - .../2018-06-20-09-12-21.bpo-23224.zxCQ13.rst | 6 - ...2-09-05-12-17-34.gh-issue-88233.gff9qJ.rst | 2 - ...2-10-22-09-26-43.gh-issue-96764.Dh9Y5L.rst | 1 - ...3-01-02-22-41-44.gh-issue-99138.17hp9U.rst | 1 - ...-01-06-21-14-41.gh-issue-100809.I697UT.rst | 3 - ...-01-25-00-14-52.gh-issue-101277.FceHX7.rst | 2 - ...-01-27-02-53-50.gh-issue-101360.bPB7SL.rst | 3 - ...3-02-01-10-42-16.gh-issue-63301.XNxSFh.rst | 1 - ...-02-04-16-35-46.gh-issue-101561.Xo6pIZ.rst | 1 - ...3-02-05-21-40-15.gh-issue-85984.Kfzbb2.rst | 4 - ...-02-07-20-46-08.gh-issue-101362.2ckZ6R.rst | 2 - ...-02-07-21-16-41.gh-issue-101362.KMQllM.rst | 2 - ...-02-07-22-20-32.gh-issue-101362.Jlk6mt.rst | 4 - ...-02-07-22-21-46.gh-issue-101446.-c0FdK.rst | 2 - ...-02-08-18-20-58.gh-issue-101693.4_LPXj.rst | 6 - ...-02-10-11-59-13.gh-issue-101773.J_kI7y.rst | 2 - ...-02-10-16-02-29.gh-issue-101517.r7S2u8.rst | 1 - ...3-02-11-13-23-29.gh-issue-97786.QjvQ1B.rst | 2 - ...3-02-13-12-55-48.gh-issue-87634.q-SBhJ.rst | 1 - ...-02-14-09-08-48.gh-issue-101892.FMos8l.rst | 3 - ...3-02-15-01-54-06.gh-issue-99108.rjTSic.rst | 3 - ...-02-17-18-44-27.gh-issue-101997.A6_blD.rst | 1 - ...3-02-17-19-00-58.gh-issue-97930.C_nQjb.rst | 4 - ...-02-17-20-24-15.gh-issue-101566.FjgWBt.rst | 4 - ...-02-21-07-15-41.gh-issue-101936.QVOxHH.rst | 2 - ...-02-21-10-05-33.gh-issue-101961.7e56jh.rst | 2 - ...-02-23-15-06-01.gh-issue-102179.P6KQ4c.rst | 1 - ...3-02-23-20-39-52.gh-issue-81652.Vxz0Mr.rst | 2 - ...3-02-26-12-37-17.gh-issue-91038.S4rFH_.rst | 1 - ...-02-28-09-52-25.gh-issue-101979.or3hXV.rst | 2 - ...-03-04-14-46-47.gh-issue-102302.-b_s6Z.rst | 1 - ...-01-24-16-12-00.gh-issue-101283.9tqu39.rst | 3 - ...3-02-08-12-57-35.gh-issue-99108.6tnmhA.rst | 4 - ...-02-08-22-03-04.gh-issue-101727.9P5eZz.rst | 4 - ...3-02-17-10-42-48.gh-issue-99108.MKA8-f.rst | 2 - ...3-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst | 4 - ...3-02-11-22-36-10.gh-issue-85984.EVXjT9.rst | 1 - ...-02-18-10-51-02.gh-issue-102019.0797SJ.rst | 2 - ...-01-25-11-33-54.gh-issue-101196.wAX_2g.rst | 3 - ...-02-07-18-22-54.gh-issue-101614.NjVP0n.rst | 1 - ...-02-09-22-09-27.gh-issue-101759.zFlqSH.rst | 1 - ...-02-10-14-26-05.gh-issue-101763.RPaj7r.rst | 1 - ...-02-13-16-32-50.gh-issue-101849.7lm_53.rst | 1 - ...-02-13-18-05-49.gh-issue-101881._TnHzN.rst | 1 - ...-02-15-11-08-10.gh-issue-101881.fScr3m.rst | 1 - ...-03-01-01-36-39.gh-issue-102344.Dgfux4.rst | 2 - ...-02-09-22-07-17.gh-issue-101759.B0JP2H.rst | 1 - README.rst | 2 +- 83 files changed, 864 insertions(+), 229 deletions(-) create mode 100644 Misc/NEWS.d/3.12.0a6.rst delete mode 100644 Misc/NEWS.d/next/Build/2022-09-14-10-38-15.gh-issue-96821.Zk2a9c.rst delete mode 100644 Misc/NEWS.d/next/Build/2022-12-18-08-33-28.gh-issue-100221.K94Ct3.rst delete mode 100644 Misc/NEWS.d/next/Build/2022-12-20-01-06-17.gh-issue-99942.lbmzYj.rst delete mode 100644 Misc/NEWS.d/next/Build/2023-01-12-00-49-16.gh-issue-99942.DUR8b4.rst delete mode 100644 Misc/NEWS.d/next/C API/2022-04-21-17-25-22.gh-issue-91744.FgvaMi.rst delete mode 100644 Misc/NEWS.d/next/C API/2023-02-06-16-14-30.gh-issue-101578.PW5fA9.rst delete mode 100644 Misc/NEWS.d/next/C API/2023-02-09-10-38-20.gh-issue-99293.mFqfpp.rst delete mode 100644 Misc/NEWS.d/next/C API/2023-02-14-15-53-01.gh-issue-101907.HgF1N2.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-11-02-20-23-47.gh-issue-98627.VJkdRM.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-01-04-12-49-33.gh-issue-100719.uRPccL.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-07-14-56-43.gh-issue-101632.Fd1yxk.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-08-17-13-31.gh-issue-101696.seJhTt.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-10-01-15-57.gh-issue-101430.T3Gegb.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-10-07-21-47.gh-issue-101765.MO5LlC.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-10-15-54-57.gh-issue-87849.IUVvPz.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-11-23-14-06.gh-issue-84783._P5sMa.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-12-22-40-22.gh-issue-101857._bribG.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-13-18-21-14.gh-issue-101799.wpHbCn.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-13-22-21-58.gh-issue-74895.esMNtq.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-16-16-57-23.gh-issue-101952.Zo1dlq.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-16-23-19-01.gh-issue-101967.Kqr1dz.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-17-10-12-13.gh-issue-100982.mJGJQw.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-20-15-18-33.gh-issue-102056.uHKuwH.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-22-15-15-32.gh-issue-102027.Km4G-d.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-24-17-59-39.gh-issue-102126.HTT8Vc.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-26-23-10-32.gh-issue-102250.7MUKoC.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-28-21-17-03.gh-issue-102336.-wL3Tm.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-04-20-56-12.gh-issue-102356.07KvUd.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-06-13-05-33.gh-issue-102416.dz6K5f.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-07-16-56-28.gh-issue-102493.gTXrcD.rst delete mode 100644 Misc/NEWS.d/next/Documentation/2023-02-07-21-43-24.gh-issue-97725.cuY7Cd.rst delete mode 100644 Misc/NEWS.d/next/Documentation/2023-02-19-10-33-01.gh-issue-85417.kYO8u3.rst delete mode 100644 Misc/NEWS.d/next/Library/2018-06-20-09-12-21.bpo-23224.zxCQ13.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-09-05-12-17-34.gh-issue-88233.gff9qJ.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-10-22-09-26-43.gh-issue-96764.Dh9Y5L.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-01-02-22-41-44.gh-issue-99138.17hp9U.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-01-06-21-14-41.gh-issue-100809.I697UT.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-01-25-00-14-52.gh-issue-101277.FceHX7.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-01-27-02-53-50.gh-issue-101360.bPB7SL.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-01-10-42-16.gh-issue-63301.XNxSFh.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-04-16-35-46.gh-issue-101561.Xo6pIZ.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-05-21-40-15.gh-issue-85984.Kfzbb2.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-07-20-46-08.gh-issue-101362.2ckZ6R.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-07-21-16-41.gh-issue-101362.KMQllM.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-07-22-20-32.gh-issue-101362.Jlk6mt.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-07-22-21-46.gh-issue-101446.-c0FdK.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-08-18-20-58.gh-issue-101693.4_LPXj.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-10-11-59-13.gh-issue-101773.J_kI7y.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-10-16-02-29.gh-issue-101517.r7S2u8.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-11-13-23-29.gh-issue-97786.QjvQ1B.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-13-12-55-48.gh-issue-87634.q-SBhJ.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-15-01-54-06.gh-issue-99108.rjTSic.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-17-18-44-27.gh-issue-101997.A6_blD.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-17-19-00-58.gh-issue-97930.C_nQjb.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-17-20-24-15.gh-issue-101566.FjgWBt.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-21-07-15-41.gh-issue-101936.QVOxHH.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-21-10-05-33.gh-issue-101961.7e56jh.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-23-15-06-01.gh-issue-102179.P6KQ4c.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-23-20-39-52.gh-issue-81652.Vxz0Mr.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-26-12-37-17.gh-issue-91038.S4rFH_.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-28-09-52-25.gh-issue-101979.or3hXV.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-04-14-46-47.gh-issue-102302.-b_s6Z.rst delete mode 100644 Misc/NEWS.d/next/Security/2023-01-24-16-12-00.gh-issue-101283.9tqu39.rst delete mode 100644 Misc/NEWS.d/next/Security/2023-02-08-12-57-35.gh-issue-99108.6tnmhA.rst delete mode 100644 Misc/NEWS.d/next/Security/2023-02-08-22-03-04.gh-issue-101727.9P5eZz.rst delete mode 100644 Misc/NEWS.d/next/Security/2023-02-17-10-42-48.gh-issue-99108.MKA8-f.rst delete mode 100644 Misc/NEWS.d/next/Tests/2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst delete mode 100644 Misc/NEWS.d/next/Tests/2023-02-11-22-36-10.gh-issue-85984.EVXjT9.rst delete mode 100644 Misc/NEWS.d/next/Tests/2023-02-18-10-51-02.gh-issue-102019.0797SJ.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-01-25-11-33-54.gh-issue-101196.wAX_2g.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-02-07-18-22-54.gh-issue-101614.NjVP0n.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-02-09-22-09-27.gh-issue-101759.zFlqSH.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-02-10-14-26-05.gh-issue-101763.RPaj7r.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-02-13-16-32-50.gh-issue-101849.7lm_53.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-02-13-18-05-49.gh-issue-101881._TnHzN.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-02-15-11-08-10.gh-issue-101881.fScr3m.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-03-01-01-36-39.gh-issue-102344.Dgfux4.rst delete mode 100644 Misc/NEWS.d/next/macOS/2023-02-09-22-07-17.gh-issue-101759.B0JP2H.rst diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 7957220ed7cf9f0..fd16421eace94af 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -20,10 +20,10 @@ #define PY_MINOR_VERSION 12 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 5 +#define PY_RELEASE_SERIAL 6 /* Version as a string */ -#define PY_VERSION "3.12.0a5+" +#define PY_VERSION "3.12.0a6" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index e7f403d3ffbf124..573065b4b714d9b 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Tue Feb 7 13:18:04 2023 +# Autogenerated by Sphinx on Tue Mar 7 22:42:28 2023 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' @@ -2499,42 +2499,33 @@ 'alive\n' 'until the next garbage collection occurs.\n' '\n' - 'Before an "except" clause’s suite is executed, details about ' - 'the\n' - 'exception are stored in the "sys" module and can be accessed ' - 'via\n' - '"sys.exc_info()". "sys.exc_info()" returns a 3-tuple consisting ' - 'of the\n' - 'exception class, the exception instance and a traceback object ' - '(see\n' - 'section The standard type hierarchy) identifying the point in ' - 'the\n' - 'program where the exception occurred. The details about the ' - 'exception\n' - 'accessed via "sys.exc_info()" are restored to their previous ' - 'values\n' - 'when leaving an exception handler:\n' + 'Before an "except" clause’s suite is executed, the exception is ' + 'stored\n' + 'in the "sys" module, where it can be accessed from within the ' + 'body of\n' + 'the "except" clause by calling "sys.exception()". When leaving ' + 'an\n' + 'exception handler, the exception stored in the "sys" module is ' + 'reset\n' + 'to its previous value:\n' '\n' - ' >>> print(sys.exc_info())\n' - ' (None, None, None)\n' + ' >>> print(sys.exception())\n' + ' None\n' ' >>> try:\n' ' ... raise TypeError\n' ' ... except:\n' - ' ... print(sys.exc_info())\n' + ' ... print(repr(sys.exception()))\n' ' ... try:\n' ' ... raise ValueError\n' ' ... except:\n' - ' ... print(sys.exc_info())\n' - ' ... print(sys.exc_info())\n' + ' ... print(repr(sys.exception()))\n' + ' ... print(repr(sys.exception()))\n' ' ...\n' - " (, TypeError(), )\n' - " (, ValueError(), )\n' - " (, TypeError(), )\n' - ' >>> print(sys.exc_info())\n' - ' (None, None, None)\n' + ' TypeError()\n' + ' ValueError()\n' + ' TypeError()\n' + ' >>> print(sys.exception())\n' + ' None\n' '\n' '\n' '"except*" clause\n' @@ -4533,7 +4524,7 @@ 'objects and\n' ' implements an "__eq__()" method, it should not ' 'implement\n' - ' "__hash__()", since the implementation of hashable ' + ' "__hash__()", since the implementation of *hashable* ' 'collections\n' ' requires that a key’s hash value is immutable (if the ' 'object’s hash\n' @@ -9453,7 +9444,7 @@ ' hashable collections. If a class defines mutable objects ' 'and\n' ' implements an "__eq__()" method, it should not implement\n' - ' "__hash__()", since the implementation of hashable ' + ' "__hash__()", since the implementation of *hashable* ' 'collections\n' ' requires that a key’s hash value is immutable (if the ' 'object’s hash\n' @@ -12644,37 +12635,31 @@ 'cycle with the stack frame, keeping all locals in that frame alive\n' 'until the next garbage collection occurs.\n' '\n' - 'Before an "except" clause’s suite is executed, details about the\n' - 'exception are stored in the "sys" module and can be accessed via\n' - '"sys.exc_info()". "sys.exc_info()" returns a 3-tuple consisting of ' - 'the\n' - 'exception class, the exception instance and a traceback object (see\n' - 'section The standard type hierarchy) identifying the point in the\n' - 'program where the exception occurred. The details about the ' - 'exception\n' - 'accessed via "sys.exc_info()" are restored to their previous values\n' - 'when leaving an exception handler:\n' + 'Before an "except" clause’s suite is executed, the exception is ' + 'stored\n' + 'in the "sys" module, where it can be accessed from within the body ' + 'of\n' + 'the "except" clause by calling "sys.exception()". When leaving an\n' + 'exception handler, the exception stored in the "sys" module is reset\n' + 'to its previous value:\n' '\n' - ' >>> print(sys.exc_info())\n' - ' (None, None, None)\n' + ' >>> print(sys.exception())\n' + ' None\n' ' >>> try:\n' ' ... raise TypeError\n' ' ... except:\n' - ' ... print(sys.exc_info())\n' + ' ... print(repr(sys.exception()))\n' ' ... try:\n' ' ... raise ValueError\n' ' ... except:\n' - ' ... print(sys.exc_info())\n' - ' ... print(sys.exc_info())\n' + ' ... print(repr(sys.exception()))\n' + ' ... print(repr(sys.exception()))\n' ' ...\n' - " (, TypeError(), )\n' - " (, ValueError(), )\n' - " (, TypeError(), )\n' - ' >>> print(sys.exc_info())\n' - ' (None, None, None)\n' + ' TypeError()\n' + ' ValueError()\n' + ' TypeError()\n' + ' >>> print(sys.exception())\n' + ' None\n' '\n' '\n' '"except*" clause\n' @@ -14408,7 +14393,7 @@ ' New in version 3.10.\n' '\n' 'Keys views are set-like since their entries are unique and ' - 'hashable.\n' + '*hashable*.\n' 'If all values are hashable, so that "(key, value)" pairs are ' 'unique\n' 'and hashable, then the items view is also set-like. (Values ' diff --git a/Misc/NEWS.d/3.12.0a6.rst b/Misc/NEWS.d/3.12.0a6.rst new file mode 100644 index 000000000000000..2bcb4c8c854d4e9 --- /dev/null +++ b/Misc/NEWS.d/3.12.0a6.rst @@ -0,0 +1,821 @@ +.. date: 2023-02-17-10-42-48 +.. gh-issue: 99108 +.. nonce: MKA8-f +.. release date: 2023-03-07 +.. section: Security + +Replace builtin hashlib implementations of MD5 and SHA1 with verified ones +from the HACL* project. + +.. + +.. date: 2023-02-08-22-03-04 +.. gh-issue: 101727 +.. nonce: 9P5eZz +.. section: Security + +Updated the OpenSSL version used in Windows and macOS binary release builds +to 1.1.1t to address CVE-2023-0286, CVE-2022-4303, and CVE-2022-4303 per +`the OpenSSL 2023-02-07 security advisory +`_. + +.. + +.. date: 2023-02-08-12-57-35 +.. gh-issue: 99108 +.. nonce: 6tnmhA +.. section: Security + +Replace the builtin :mod:`hashlib` implementations of SHA2-384 and SHA2-512 +originally from LibTomCrypt with formally verified, side-channel resistant +code from the `HACL* `_ project. +The builtins remain a fallback only used when OpenSSL does not provide them. + +.. + +.. date: 2023-01-24-16-12-00 +.. gh-issue: 101283 +.. nonce: 9tqu39 +.. section: Security + +:class:`subprocess.Popen` now uses a safer approach to find ``cmd.exe`` when +launching with ``shell=True``. Patch by Eryk Sun, based on a patch by Oleg +Iarygin. + +.. + +.. date: 2023-03-07-16-56-28 +.. gh-issue: 102493 +.. nonce: gTXrcD +.. section: Core and Builtins + +Fix regression in semantics of normalisation in ``PyErr_SetObject``. + +.. + +.. date: 2023-03-06-13-05-33 +.. gh-issue: 102416 +.. nonce: dz6K5f +.. section: Core and Builtins + +Do not memoize incorrectly automatically generated loop rules in the parser. +Patch by Pablo Galindo. + +.. + +.. date: 2023-03-04-20-56-12 +.. gh-issue: 102356 +.. nonce: 07KvUd +.. section: Core and Builtins + +Fix a bug that caused a crash when deallocating deeply nested filter +objects. Patch by Marta Gómez Macías. + +.. + +.. date: 2023-02-28-21-17-03 +.. gh-issue: 102336 +.. nonce: -wL3Tm +.. section: Core and Builtins + +Cleanup Windows 7 specific special handling. Patch by Max Bachmann. + +.. + +.. date: 2023-02-26-23-10-32 +.. gh-issue: 102250 +.. nonce: 7MUKoC +.. section: Core and Builtins + +Fixed a segfault occurring when the interpreter calls a ``__bool__`` method +that raises. + +.. + +.. date: 2023-02-24-17-59-39 +.. gh-issue: 102126 +.. nonce: HTT8Vc +.. section: Core and Builtins + +Fix deadlock at shutdown when clearing thread states if any finalizer tries +to acquire the runtime head lock. Patch by Kumar Aditya. + +.. + +.. date: 2023-02-22-15-15-32 +.. gh-issue: 102027 +.. nonce: Km4G-d +.. section: Core and Builtins + +Use ``GetCurrentProcessId`` on Windows when ``getpid`` is unavailable. Patch +by Max Bachmann. + +.. + +.. date: 2023-02-20-15-18-33 +.. gh-issue: 102056 +.. nonce: uHKuwH +.. section: Core and Builtins + +Fix error handling bugs in interpreter's exception printing code, which +could cause a crash on infinite recursion. + +.. + +.. date: 2023-02-17-10-12-13 +.. gh-issue: 100982 +.. nonce: mJGJQw +.. section: Core and Builtins + +Restrict the scope of the :opcode:`FOR_ITER_RANGE` instruction to the scope +of the original :opcode:`FOR_ITER` instruction, to allow instrumentation. + +.. + +.. date: 2023-02-16-23-19-01 +.. gh-issue: 101967 +.. nonce: Kqr1dz +.. section: Core and Builtins + +Fix possible segfault in ``positional_only_passed_as_keyword`` function, +when new list created. + +.. + +.. date: 2023-02-16-16-57-23 +.. gh-issue: 101952 +.. nonce: Zo1dlq +.. section: Core and Builtins + +Fix possible segfault in ``BUILD_SET`` opcode, when new set created. + +.. + +.. date: 2023-02-13-22-21-58 +.. gh-issue: 74895 +.. nonce: esMNtq +.. section: Core and Builtins + +:mod:`socket.getaddrinfo` no longer raises :class:`OverflowError` for +:class:`int` **port** values outside of the C long range. Out of range +values are left up to the underlying string based C library API to report. A +:class:`socket.gaierror` ``SAI_SERVICE`` may occur instead, or no error at +all as not all platform C libraries generate an error. + +.. + +.. date: 2023-02-13-18-21-14 +.. gh-issue: 101799 +.. nonce: wpHbCn +.. section: Core and Builtins + +Add :opcode:`CALL_INTRINSIC_2` and use it instead of +:opcode:`PREP_RERAISE_STAR`. + +.. + +.. date: 2023-02-12-22-40-22 +.. gh-issue: 101857 +.. nonce: _bribG +.. section: Core and Builtins + +Fix xattr support detection on Linux systems by widening the check to linux, +not just glibc. This fixes support for musl. + +.. + +.. date: 2023-02-11-23-14-06 +.. gh-issue: 84783 +.. nonce: _P5sMa +.. section: Core and Builtins + +Make the slice object hashable. + +.. + +.. date: 2023-02-10-15-54-57 +.. gh-issue: 87849 +.. nonce: IUVvPz +.. section: Core and Builtins + +Change the ``SEND`` instruction to leave the receiver on the stack. This +allows the specialized form of ``SEND`` to skip the chain of C calls and +jump directly to the ``RESUME`` in the generator or coroutine. + +.. + +.. date: 2023-02-10-07-21-47 +.. gh-issue: 101765 +.. nonce: MO5LlC +.. section: Core and Builtins + +Fix SystemError / segmentation fault in iter ``__reduce__`` when internal +access of ``builtins.__dict__`` keys mutates the iter object. + +.. + +.. date: 2023-02-10-01-15-57 +.. gh-issue: 101430 +.. nonce: T3Gegb +.. section: Core and Builtins + +Update :mod:`tracemalloc` to handle presize of object properly. Patch by +Dong-hee Na. + +.. + +.. date: 2023-02-08-17-13-31 +.. gh-issue: 101696 +.. nonce: seJhTt +.. section: Core and Builtins + +Invalidate type version tag in ``_PyStaticType_Dealloc`` for static types, +avoiding bug where a false cache hit could crash the interpreter. Patch by +Kumar Aditya. + +.. + +.. date: 2023-02-07-14-56-43 +.. gh-issue: 101632 +.. nonce: Fd1yxk +.. section: Core and Builtins + +Adds a new :opcode:`RETURN_CONST` instruction. + +.. + +.. date: 2023-01-04-12-49-33 +.. gh-issue: 100719 +.. nonce: uRPccL +.. section: Core and Builtins + +Remove gi_code field from generator (and coroutine and async generator) +objects as it is redundant. The frame already includes a reference to the +code object. + +.. + +.. date: 2022-11-02-20-23-47 +.. gh-issue: 98627 +.. nonce: VJkdRM +.. section: Core and Builtins + +When an interpreter is configured to check (and only then), importing an +extension module will now fail when the extension does not support multiple +interpreters (i.e. doesn't implement PEP 489 multi-phase init). This does +not apply to the main interpreter, nor to subinterpreters created with +``Py_NewInterpreter()``. + +.. + +.. date: 2023-03-04-14-46-47 +.. gh-issue: 102302 +.. nonce: -b_s6Z +.. section: Library + +Micro-optimise hashing of :class:`inspect.Parameter`, reducing the time it +takes to hash an instance by around 40%. + +.. + +.. date: 2023-02-28-09-52-25 +.. gh-issue: 101979 +.. nonce: or3hXV +.. section: Library + +Fix a bug where parentheses in the ``metavar`` argument to +:meth:`argparse.ArgumentParser.add_argument` were dropped. Patch by Yeojin +Kim. + +.. + +.. date: 2023-02-26-12-37-17 +.. gh-issue: 91038 +.. nonce: S4rFH_ +.. section: Library + +:meth:`platform.platform` now has boolean default arguments. + +.. + +.. date: 2023-02-23-20-39-52 +.. gh-issue: 81652 +.. nonce: Vxz0Mr +.. section: Library + +Add :data:`mmap.MAP_ALIGNED_SUPER` FreeBSD and :data:`mmap.MAP_CONCEAL` +OpenBSD constants to :mod:`mmap`. Patch by Yeojin Kim. + +.. + +.. date: 2023-02-23-15-06-01 +.. gh-issue: 102179 +.. nonce: P6KQ4c +.. section: Library + +Fix :func:`os.dup2` error message for negative fds. + +.. + +.. date: 2023-02-21-10-05-33 +.. gh-issue: 101961 +.. nonce: 7e56jh +.. section: Library + +For the binary mode, :func:`fileinput.hookcompressed` doesn't set the +``encoding`` value even if the value is ``None``. Patch by Gihwan Kim. + +.. + +.. date: 2023-02-21-07-15-41 +.. gh-issue: 101936 +.. nonce: QVOxHH +.. section: Library + +The default value of ``fp`` becomes :class:`io.BytesIO` if +:exc:`~urllib.error.HTTPError` is initialized without a designated ``fp`` +parameter. Patch by Long Vo. + +.. + +.. date: 2023-02-17-20-24-15 +.. gh-issue: 101566 +.. nonce: FjgWBt +.. section: Library + +In zipfile, sync Path with `zipp 3.14 +`_, including +fix for extractall on the underlying zipfile after being wrapped in +``Path``. + +.. + +.. date: 2023-02-17-19-00-58 +.. gh-issue: 97930 +.. nonce: C_nQjb +.. section: Library + +Apply changes from `importlib_resources 5.12 +`_, +including fix for ``MultiplexedPath`` to support directories in multiple +namespaces (python/importlib_resources#265). + +.. + +.. date: 2023-02-17-18-44-27 +.. gh-issue: 101997 +.. nonce: A6_blD +.. section: Library + +Upgrade pip wheel bundled with ensurepip (pip 23.0.1) + +.. + +.. date: 2023-02-15-01-54-06 +.. gh-issue: 99108 +.. nonce: rjTSic +.. section: Library + +The built-in extension modules for :mod:`hashlib` SHA2 algorithms, used when +OpenSSL does not provide them, now live in a single internal ``_sha2`` +module instead of separate ``_sha256`` and ``_sha512`` modules. + +.. + +.. date: 2023-02-14-09-08-48 +.. gh-issue: 101892 +.. nonce: FMos8l +.. section: Library + +Callable iterators no longer raise :class:`SystemError` when the callable +object exhausts the iterator but forgets to either return a sentinel value +or raise :class:`StopIteration`. + +.. + +.. date: 2023-02-13-12-55-48 +.. gh-issue: 87634 +.. nonce: q-SBhJ +.. section: Library + +Remove locking behavior from :func:`functools.cached_property`. + +.. + +.. date: 2023-02-11-13-23-29 +.. gh-issue: 97786 +.. nonce: QjvQ1B +.. section: Library + +Fix potential undefined behaviour in corner cases of floating-point-to-time +conversions. + +.. + +.. date: 2023-02-10-16-02-29 +.. gh-issue: 101517 +.. nonce: r7S2u8 +.. section: Library + +Fixed bug where :mod:`bdb` looks up the source line with :mod:`linecache` +with a ``lineno=None``, which causes it to fail with an unhandled exception. + +.. + +.. date: 2023-02-10-11-59-13 +.. gh-issue: 101773 +.. nonce: J_kI7y +.. section: Library + +Optimize :class:`fractions.Fraction` for small components. The private +argument ``_normalize`` of the :class:`fractions.Fraction` constructor has +been removed. + +.. + +.. date: 2023-02-08-18-20-58 +.. gh-issue: 101693 +.. nonce: 4_LPXj +.. section: Library + +In :meth:`sqlite3.Cursor.execute`, :exc:`DeprecationWarning` is now emitted +when :ref:`named placeholders ` are used together with +parameters supplied as a :term:`sequence` instead of as a :class:`dict`. +Starting from Python 3.14, using named placeholders with parameters supplied +as a sequence will raise a :exc:`~sqlite3.ProgrammingError`. Patch by Erlend +E. Aasland. + +.. + +.. date: 2023-02-07-22-21-46 +.. gh-issue: 101446 +.. nonce: -c0FdK +.. section: Library + +Change repr of :class:`collections.OrderedDict` to use regular dictionary +formating instead of pairs of keys and values. + +.. + +.. date: 2023-02-07-22-20-32 +.. gh-issue: 101362 +.. nonce: Jlk6mt +.. section: Library + +Speed up :class:`pathlib.PurePath` construction by handling arguments more +uniformly. When a :class:`pathlib.Path` argument is supplied, we use its +string representation rather than joining its parts with +:func:`os.path.join`. + +.. + +.. date: 2023-02-07-21-16-41 +.. gh-issue: 101362 +.. nonce: KMQllM +.. section: Library + +Speed up :class:`pathlib.PurePath` construction by calling +:func:`os.path.join` only when two or more arguments are given. + +.. + +.. date: 2023-02-07-20-46-08 +.. gh-issue: 101362 +.. nonce: 2ckZ6R +.. section: Library + +Speed up :class:`pathlib.Path` construction by running the path flavour +compatibility check only when pathlib is imported. + +.. + +.. date: 2023-02-05-21-40-15 +.. gh-issue: 85984 +.. nonce: Kfzbb2 +.. section: Library + +Refactored the implementation of :func:`pty.fork` to use +:func:`os.login_tty`. + +A :exc:`DeprecationWarning` is now raised by ``pty.master_open()`` and +``pty.slave_open()``. They were undocumented and deprecated long long ago in +the docstring in favor of :func:`pty.openpty`. + +.. + +.. date: 2023-02-04-16-35-46 +.. gh-issue: 101561 +.. nonce: Xo6pIZ +.. section: Library + +Add a new decorator :func:`typing.override`. See :pep:`698` for details. +Patch by Steven Troxler. + +.. + +.. date: 2023-02-01-10-42-16 +.. gh-issue: 63301 +.. nonce: XNxSFh +.. section: Library + +Set exit code when :mod:`tabnanny` CLI exits on error. + +.. + +.. date: 2023-01-27-02-53-50 +.. gh-issue: 101360 +.. nonce: bPB7SL +.. section: Library + +Fix anchor matching in :meth:`pathlib.PureWindowsPath.match`. Path and +pattern anchors are now matched with :mod:`fnmatch`, just like other path +parts. This allows patterns such as ``"*:/Users/*"`` to be matched. + +.. + +.. date: 2023-01-25-00-14-52 +.. gh-issue: 101277 +.. nonce: FceHX7 +.. section: Library + +Remove global state from :mod:`itertools` module (:pep:`687`). Patches by +Erlend E. Aasland. + +.. + +.. date: 2023-01-06-21-14-41 +.. gh-issue: 100809 +.. nonce: I697UT +.. section: Library + +Fix handling of drive-relative paths (like 'C:' and 'C:foo') in +:meth:`pathlib.Path.absolute`. This method now uses the OS API to retrieve +the correct current working directory for the drive. + +.. + +.. date: 2023-01-02-22-41-44 +.. gh-issue: 99138 +.. nonce: 17hp9U +.. section: Library + +Apply :pep:`687` to :mod:`zoneinfo`. Patch by Erlend E. Aasland. + +.. + +.. date: 2022-10-22-09-26-43 +.. gh-issue: 96764 +.. nonce: Dh9Y5L +.. section: Library + +:func:`asyncio.wait_for` now uses :func:`asyncio.timeout` as its underlying +implementation. Patch by Kumar Aditya. + +.. + +.. date: 2022-09-05-12-17-34 +.. gh-issue: 88233 +.. nonce: gff9qJ +.. section: Library + +Correctly preserve "extra" fields in ``zipfile`` regardless of their +ordering relative to a zip64 "extra." + +.. + +.. bpo: 23224 +.. date: 2018-06-20-09-12-21 +.. nonce: zxCQ13 +.. section: Library + +Fix segfaults when creating :class:`lzma.LZMADecompressor` and +:class:`bz2.BZ2Decompressor` objects without calling ``__init__()``, and fix +leakage of locks and internal buffers when calling the ``__init__()`` +methods of :class:`lzma.LZMADecompressor`, :class:`lzma.LZMACompressor`, +:class:`bz2.BZ2Compressor`, and :class:`bz2.BZ2Decompressor` objects +multiple times. + +.. + +.. date: 2023-02-19-10-33-01 +.. gh-issue: 85417 +.. nonce: kYO8u3 +.. section: Documentation + +Update :mod:`cmath` documentation to clarify behaviour on branch cuts. + +.. + +.. date: 2023-02-07-21-43-24 +.. gh-issue: 97725 +.. nonce: cuY7Cd +.. section: Documentation + +Fix :meth:`asyncio.Task.print_stack` description for ``file=None``. Patch by +Oleg Iarygin. + +.. + +.. date: 2023-02-18-10-51-02 +.. gh-issue: 102019 +.. nonce: 0797SJ +.. section: Tests + +Fix deadlock on shutdown if ``test_current_{exception,frames}`` fails. Patch +by Jacob Bower. + +.. + +.. date: 2023-02-11-22-36-10 +.. gh-issue: 85984 +.. nonce: EVXjT9 +.. section: Tests + +Utilize new "winsize" functions from termios in pty tests. + +.. + +.. date: 2023-02-11-20-28-08 +.. gh-issue: 89792 +.. nonce: S-Y5BZ +.. section: Tests + +``test_tools`` now copies up to 10x less source data to a temporary +directory during the ``freeze`` test by ignoring git metadata and other +artifacts. It also limits its python build parallelism based on +os.cpu_count instead of hard coding it as 8 cores. + +.. + +.. date: 2023-01-12-00-49-16 +.. gh-issue: 99942 +.. nonce: DUR8b4 +.. section: Build + +On Android, in a static build, python-config in embed mode no longer +incorrectly reports a library to link to. + +.. + +.. date: 2022-12-20-01-06-17 +.. gh-issue: 99942 +.. nonce: lbmzYj +.. section: Build + +On Android, python.pc now correctly reports the library to link to, the same +as python-config.sh. + +.. + +.. date: 2022-12-18-08-33-28 +.. gh-issue: 100221 +.. nonce: K94Ct3 +.. section: Build + +Fix creating install directories in ``make sharedinstall`` if they exist +outside ``DESTDIR`` already. + +.. + +.. date: 2022-09-14-10-38-15 +.. gh-issue: 96821 +.. nonce: Zk2a9c +.. section: Build + +Explicitly mark C extension modules that need defined signed integer +overflow, and add a configure option :option:`--with-strict-overflow`. Patch +by Matthias Görgens and Shantanu Jain. + +.. + +.. date: 2023-03-01-01-36-39 +.. gh-issue: 102344 +.. nonce: Dgfux4 +.. section: Windows + +Implement ``winreg.QueryValue`` using ``QueryValueEx`` and +``winreg.SetValue`` using ``SetValueEx``. Patch by Max Bachmann. + +.. + +.. date: 2023-02-15-11-08-10 +.. gh-issue: 101881 +.. nonce: fScr3m +.. section: Windows + +Handle read and write operations on non-blocking pipes properly on Windows. + +.. + +.. date: 2023-02-13-18-05-49 +.. gh-issue: 101881 +.. nonce: _TnHzN +.. section: Windows + +Add support for the os.get_blocking() and os.set_blocking() functions on +Windows. + +.. + +.. date: 2023-02-13-16-32-50 +.. gh-issue: 101849 +.. nonce: 7lm_53 +.. section: Windows + +Ensures installer will correctly upgrade existing ``py.exe`` launcher +installs. + +.. + +.. date: 2023-02-10-14-26-05 +.. gh-issue: 101763 +.. nonce: RPaj7r +.. section: Windows + +Updates copy of libffi bundled with Windows installs to 3.4.4. + +.. + +.. date: 2023-02-09-22-09-27 +.. gh-issue: 101759 +.. nonce: zFlqSH +.. section: Windows + +Update Windows installer to SQLite 3.40.1. + +.. + +.. date: 2023-02-07-18-22-54 +.. gh-issue: 101614 +.. nonce: NjVP0n +.. section: Windows + +Correctly handle extensions built against debug binaries that reference +``python3_d.dll``. + +.. + +.. date: 2023-01-25-11-33-54 +.. gh-issue: 101196 +.. nonce: wAX_2g +.. section: Windows + +The functions ``os.path.isdir``, ``os.path.isfile``, ``os.path.islink`` and +``os.path.exists`` are now 13% to 28% faster on Windows, by making fewer +Win32 API calls. + +.. + +.. date: 2023-02-09-22-07-17 +.. gh-issue: 101759 +.. nonce: B0JP2H +.. section: macOS + +Update macOS installer to SQLite 3.40.1. + +.. + +.. date: 2023-02-14-15-53-01 +.. gh-issue: 101907 +.. nonce: HgF1N2 +.. section: C API + +Removes use of non-standard C++ extension in public header files. + +.. + +.. date: 2023-02-09-10-38-20 +.. gh-issue: 99293 +.. nonce: mFqfpp +.. section: C API + +Document that the Py_TPFLAGS_VALID_VERSION_TAG is an internal feature, +should not be used, and will be removed. + +.. + +.. date: 2023-02-06-16-14-30 +.. gh-issue: 101578 +.. nonce: PW5fA9 +.. section: C API + +Add :c:func:`PyErr_GetRaisedException` and +:c:func:`PyErr_SetRaisedException` for saving and restoring the current +exception. These functions return and accept a single exception object, +rather than the triple arguments of the now-deprecated :c:func:`PyErr_Fetch` +and :c:func:`PyErr_Restore`. This is less error prone and a bit more +efficient. + +Add :c:func:`PyException_GetArgs` and :c:func:`PyException_SetArgs` as +convenience functions for retrieving and modifying the +:attr:`~BaseException.args` passed to the exception's constructor. + +.. + +.. date: 2022-04-21-17-25-22 +.. gh-issue: 91744 +.. nonce: FgvaMi +.. section: C API + +Introduced the *Unstable C API tier*, marking APi that is allowed to change +in minor releases without a deprecation period. See :pep:`689` for details. diff --git a/Misc/NEWS.d/next/Build/2022-09-14-10-38-15.gh-issue-96821.Zk2a9c.rst b/Misc/NEWS.d/next/Build/2022-09-14-10-38-15.gh-issue-96821.Zk2a9c.rst deleted file mode 100644 index 865cfde8b063598..000000000000000 --- a/Misc/NEWS.d/next/Build/2022-09-14-10-38-15.gh-issue-96821.Zk2a9c.rst +++ /dev/null @@ -1,3 +0,0 @@ -Explicitly mark C extension modules that need defined signed integer overflow, -and add a configure option :option:`--with-strict-overflow`. -Patch by Matthias Görgens and Shantanu Jain. diff --git a/Misc/NEWS.d/next/Build/2022-12-18-08-33-28.gh-issue-100221.K94Ct3.rst b/Misc/NEWS.d/next/Build/2022-12-18-08-33-28.gh-issue-100221.K94Ct3.rst deleted file mode 100644 index 27c948330cfc17b..000000000000000 --- a/Misc/NEWS.d/next/Build/2022-12-18-08-33-28.gh-issue-100221.K94Ct3.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix creating install directories in ``make sharedinstall`` if they exist -outside ``DESTDIR`` already. diff --git a/Misc/NEWS.d/next/Build/2022-12-20-01-06-17.gh-issue-99942.lbmzYj.rst b/Misc/NEWS.d/next/Build/2022-12-20-01-06-17.gh-issue-99942.lbmzYj.rst deleted file mode 100644 index 63a640a9cdf7338..000000000000000 --- a/Misc/NEWS.d/next/Build/2022-12-20-01-06-17.gh-issue-99942.lbmzYj.rst +++ /dev/null @@ -1,2 +0,0 @@ -On Android, python.pc now correctly reports the library to link to, the same -as python-config.sh. diff --git a/Misc/NEWS.d/next/Build/2023-01-12-00-49-16.gh-issue-99942.DUR8b4.rst b/Misc/NEWS.d/next/Build/2023-01-12-00-49-16.gh-issue-99942.DUR8b4.rst deleted file mode 100644 index 5b692c3cc458c56..000000000000000 --- a/Misc/NEWS.d/next/Build/2023-01-12-00-49-16.gh-issue-99942.DUR8b4.rst +++ /dev/null @@ -1,2 +0,0 @@ -On Android, in a static build, python-config in embed mode no longer -incorrectly reports a library to link to. diff --git a/Misc/NEWS.d/next/C API/2022-04-21-17-25-22.gh-issue-91744.FgvaMi.rst b/Misc/NEWS.d/next/C API/2022-04-21-17-25-22.gh-issue-91744.FgvaMi.rst deleted file mode 100644 index 20db25ddd0c1f6d..000000000000000 --- a/Misc/NEWS.d/next/C API/2022-04-21-17-25-22.gh-issue-91744.FgvaMi.rst +++ /dev/null @@ -1,3 +0,0 @@ -Introduced the *Unstable C API tier*, marking APi that is allowed to change -in minor releases without a deprecation period. -See :pep:`689` for details. diff --git a/Misc/NEWS.d/next/C API/2023-02-06-16-14-30.gh-issue-101578.PW5fA9.rst b/Misc/NEWS.d/next/C API/2023-02-06-16-14-30.gh-issue-101578.PW5fA9.rst deleted file mode 100644 index 27294a9e5179c48..000000000000000 --- a/Misc/NEWS.d/next/C API/2023-02-06-16-14-30.gh-issue-101578.PW5fA9.rst +++ /dev/null @@ -1,10 +0,0 @@ -Add :c:func:`PyErr_GetRaisedException` and :c:func:`PyErr_SetRaisedException` -for saving and restoring the current exception. -These functions return and accept a single exception object, -rather than the triple arguments of the now-deprecated -:c:func:`PyErr_Fetch` and :c:func:`PyErr_Restore`. -This is less error prone and a bit more efficient. - -Add :c:func:`PyException_GetArgs` and :c:func:`PyException_SetArgs` -as convenience functions for retrieving and modifying -the :attr:`~BaseException.args` passed to the exception's constructor. diff --git a/Misc/NEWS.d/next/C API/2023-02-09-10-38-20.gh-issue-99293.mFqfpp.rst b/Misc/NEWS.d/next/C API/2023-02-09-10-38-20.gh-issue-99293.mFqfpp.rst deleted file mode 100644 index 8c0f05543747dc0..000000000000000 --- a/Misc/NEWS.d/next/C API/2023-02-09-10-38-20.gh-issue-99293.mFqfpp.rst +++ /dev/null @@ -1,2 +0,0 @@ -Document that the Py_TPFLAGS_VALID_VERSION_TAG is an internal feature, -should not be used, and will be removed. diff --git a/Misc/NEWS.d/next/C API/2023-02-14-15-53-01.gh-issue-101907.HgF1N2.rst b/Misc/NEWS.d/next/C API/2023-02-14-15-53-01.gh-issue-101907.HgF1N2.rst deleted file mode 100644 index cfc0d72cdbca003..000000000000000 --- a/Misc/NEWS.d/next/C API/2023-02-14-15-53-01.gh-issue-101907.HgF1N2.rst +++ /dev/null @@ -1 +0,0 @@ -Removes use of non-standard C++ extension in public header files. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-02-20-23-47.gh-issue-98627.VJkdRM.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-02-20-23-47.gh-issue-98627.VJkdRM.rst deleted file mode 100644 index 3d2d6f6eb0c41f5..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-02-20-23-47.gh-issue-98627.VJkdRM.rst +++ /dev/null @@ -1,5 +0,0 @@ -When an interpreter is configured to check (and only then), importing an -extension module will now fail when the extension does not support multiple -interpreters (i.e. doesn't implement PEP 489 multi-phase init). This does -not apply to the main interpreter, nor to subinterpreters created with -``Py_NewInterpreter()``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-01-04-12-49-33.gh-issue-100719.uRPccL.rst b/Misc/NEWS.d/next/Core and Builtins/2023-01-04-12-49-33.gh-issue-100719.uRPccL.rst deleted file mode 100644 index 2addef27b8ea4dc..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-01-04-12-49-33.gh-issue-100719.uRPccL.rst +++ /dev/null @@ -1,3 +0,0 @@ -Remove gi_code field from generator (and coroutine and async generator) -objects as it is redundant. The frame already includes a reference to the -code object. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-07-14-56-43.gh-issue-101632.Fd1yxk.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-07-14-56-43.gh-issue-101632.Fd1yxk.rst deleted file mode 100644 index 136909ca6999030..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-07-14-56-43.gh-issue-101632.Fd1yxk.rst +++ /dev/null @@ -1 +0,0 @@ -Adds a new :opcode:`RETURN_CONST` instruction. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-08-17-13-31.gh-issue-101696.seJhTt.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-08-17-13-31.gh-issue-101696.seJhTt.rst deleted file mode 100644 index ff2bbb4b5642522..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-08-17-13-31.gh-issue-101696.seJhTt.rst +++ /dev/null @@ -1 +0,0 @@ -Invalidate type version tag in ``_PyStaticType_Dealloc`` for static types, avoiding bug where a false cache hit could crash the interpreter. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-10-01-15-57.gh-issue-101430.T3Gegb.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-10-01-15-57.gh-issue-101430.T3Gegb.rst deleted file mode 100644 index e617d85242144ea..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-10-01-15-57.gh-issue-101430.T3Gegb.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update :mod:`tracemalloc` to handle presize of object properly. Patch by -Dong-hee Na. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-10-07-21-47.gh-issue-101765.MO5LlC.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-10-07-21-47.gh-issue-101765.MO5LlC.rst deleted file mode 100644 index cc99779a944ec6d..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-10-07-21-47.gh-issue-101765.MO5LlC.rst +++ /dev/null @@ -1 +0,0 @@ -Fix SystemError / segmentation fault in iter ``__reduce__`` when internal access of ``builtins.__dict__`` keys mutates the iter object. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-10-15-54-57.gh-issue-87849.IUVvPz.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-10-15-54-57.gh-issue-87849.IUVvPz.rst deleted file mode 100644 index da5f3ff79fd575c..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-10-15-54-57.gh-issue-87849.IUVvPz.rst +++ /dev/null @@ -1,3 +0,0 @@ -Change the ``SEND`` instruction to leave the receiver on the stack. This -allows the specialized form of ``SEND`` to skip the chain of C calls and jump -directly to the ``RESUME`` in the generator or coroutine. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-11-23-14-06.gh-issue-84783._P5sMa.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-11-23-14-06.gh-issue-84783._P5sMa.rst deleted file mode 100644 index e1c851a0825a7f0..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-11-23-14-06.gh-issue-84783._P5sMa.rst +++ /dev/null @@ -1 +0,0 @@ -Make the slice object hashable. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-12-22-40-22.gh-issue-101857._bribG.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-12-22-40-22.gh-issue-101857._bribG.rst deleted file mode 100644 index 832cc300fa9433a..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-12-22-40-22.gh-issue-101857._bribG.rst +++ /dev/null @@ -1 +0,0 @@ -Fix xattr support detection on Linux systems by widening the check to linux, not just glibc. This fixes support for musl. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-13-18-21-14.gh-issue-101799.wpHbCn.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-13-18-21-14.gh-issue-101799.wpHbCn.rst deleted file mode 100644 index 3233a573be7acdf..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-13-18-21-14.gh-issue-101799.wpHbCn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add :opcode:`CALL_INTRINSIC_2` and use it instead of -:opcode:`PREP_RERAISE_STAR`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-13-22-21-58.gh-issue-74895.esMNtq.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-13-22-21-58.gh-issue-74895.esMNtq.rst deleted file mode 100644 index adbbb601634a602..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-13-22-21-58.gh-issue-74895.esMNtq.rst +++ /dev/null @@ -1,5 +0,0 @@ -:mod:`socket.getaddrinfo` no longer raises :class:`OverflowError` for -:class:`int` **port** values outside of the C long range. Out of range values -are left up to the underlying string based C library API to report. A -:class:`socket.gaierror` ``SAI_SERVICE`` may occur instead, or no error at all -as not all platform C libraries generate an error. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-16-16-57-23.gh-issue-101952.Zo1dlq.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-16-16-57-23.gh-issue-101952.Zo1dlq.rst deleted file mode 100644 index 3902c988c8bf9ff..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-16-16-57-23.gh-issue-101952.Zo1dlq.rst +++ /dev/null @@ -1 +0,0 @@ -Fix possible segfault in ``BUILD_SET`` opcode, when new set created. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-16-23-19-01.gh-issue-101967.Kqr1dz.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-16-23-19-01.gh-issue-101967.Kqr1dz.rst deleted file mode 100644 index 6e681f910f53597..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-16-23-19-01.gh-issue-101967.Kqr1dz.rst +++ /dev/null @@ -1 +0,0 @@ -Fix possible segfault in ``positional_only_passed_as_keyword`` function, when new list created. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-17-10-12-13.gh-issue-100982.mJGJQw.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-17-10-12-13.gh-issue-100982.mJGJQw.rst deleted file mode 100644 index 53bbc860c53f374..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-17-10-12-13.gh-issue-100982.mJGJQw.rst +++ /dev/null @@ -1,2 +0,0 @@ -Restrict the scope of the :opcode:`FOR_ITER_RANGE` instruction to the scope of the -original :opcode:`FOR_ITER` instruction, to allow instrumentation. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-20-15-18-33.gh-issue-102056.uHKuwH.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-20-15-18-33.gh-issue-102056.uHKuwH.rst deleted file mode 100644 index 78cd525b365fe5e..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-20-15-18-33.gh-issue-102056.uHKuwH.rst +++ /dev/null @@ -1 +0,0 @@ -Fix error handling bugs in interpreter's exception printing code, which could cause a crash on infinite recursion. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-22-15-15-32.gh-issue-102027.Km4G-d.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-22-15-15-32.gh-issue-102027.Km4G-d.rst deleted file mode 100644 index 514a8ef26594dce..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-22-15-15-32.gh-issue-102027.Km4G-d.rst +++ /dev/null @@ -1,2 +0,0 @@ -Use ``GetCurrentProcessId`` on Windows when ``getpid`` is unavailable. Patch by -Max Bachmann. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-24-17-59-39.gh-issue-102126.HTT8Vc.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-24-17-59-39.gh-issue-102126.HTT8Vc.rst deleted file mode 100644 index 68c43688c3df035..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-24-17-59-39.gh-issue-102126.HTT8Vc.rst +++ /dev/null @@ -1 +0,0 @@ -Fix deadlock at shutdown when clearing thread states if any finalizer tries to acquire the runtime head lock. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-26-23-10-32.gh-issue-102250.7MUKoC.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-26-23-10-32.gh-issue-102250.7MUKoC.rst deleted file mode 100644 index 17ab0cd43679912..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-26-23-10-32.gh-issue-102250.7MUKoC.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed a segfault occurring when the interpreter calls a ``__bool__`` method that raises. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-28-21-17-03.gh-issue-102336.-wL3Tm.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-28-21-17-03.gh-issue-102336.-wL3Tm.rst deleted file mode 100644 index 0c3e4bd4b860948..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-28-21-17-03.gh-issue-102336.-wL3Tm.rst +++ /dev/null @@ -1 +0,0 @@ -Cleanup Windows 7 specific special handling. Patch by Max Bachmann. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-04-20-56-12.gh-issue-102356.07KvUd.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-04-20-56-12.gh-issue-102356.07KvUd.rst deleted file mode 100644 index c03fd5266bc3017..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-04-20-56-12.gh-issue-102356.07KvUd.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a bug that caused a crash when deallocating deeply nested filter -objects. Patch by Marta Gómez Macías. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-06-13-05-33.gh-issue-102416.dz6K5f.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-06-13-05-33.gh-issue-102416.dz6K5f.rst deleted file mode 100644 index 9ffc67cfb7ed56a..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-06-13-05-33.gh-issue-102416.dz6K5f.rst +++ /dev/null @@ -1 +0,0 @@ -Do not memoize incorrectly automatically generated loop rules in the parser. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-07-16-56-28.gh-issue-102493.gTXrcD.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-07-16-56-28.gh-issue-102493.gTXrcD.rst deleted file mode 100644 index 4c4e88ca4e7c3cd..000000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-07-16-56-28.gh-issue-102493.gTXrcD.rst +++ /dev/null @@ -1 +0,0 @@ -Fix regression in semantics of normalisation in ``PyErr_SetObject``. diff --git a/Misc/NEWS.d/next/Documentation/2023-02-07-21-43-24.gh-issue-97725.cuY7Cd.rst b/Misc/NEWS.d/next/Documentation/2023-02-07-21-43-24.gh-issue-97725.cuY7Cd.rst deleted file mode 100644 index fd9ea049c239687..000000000000000 --- a/Misc/NEWS.d/next/Documentation/2023-02-07-21-43-24.gh-issue-97725.cuY7Cd.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :meth:`asyncio.Task.print_stack` description for ``file=None``. -Patch by Oleg Iarygin. diff --git a/Misc/NEWS.d/next/Documentation/2023-02-19-10-33-01.gh-issue-85417.kYO8u3.rst b/Misc/NEWS.d/next/Documentation/2023-02-19-10-33-01.gh-issue-85417.kYO8u3.rst deleted file mode 100644 index a5532df14795d22..000000000000000 --- a/Misc/NEWS.d/next/Documentation/2023-02-19-10-33-01.gh-issue-85417.kYO8u3.rst +++ /dev/null @@ -1 +0,0 @@ -Update :mod:`cmath` documentation to clarify behaviour on branch cuts. diff --git a/Misc/NEWS.d/next/Library/2018-06-20-09-12-21.bpo-23224.zxCQ13.rst b/Misc/NEWS.d/next/Library/2018-06-20-09-12-21.bpo-23224.zxCQ13.rst deleted file mode 100644 index 8909753c7f9ee62..000000000000000 --- a/Misc/NEWS.d/next/Library/2018-06-20-09-12-21.bpo-23224.zxCQ13.rst +++ /dev/null @@ -1,6 +0,0 @@ -Fix segfaults when creating :class:`lzma.LZMADecompressor` and -:class:`bz2.BZ2Decompressor` objects without calling ``__init__()``, and fix -leakage of locks and internal buffers when calling the ``__init__()`` -methods of :class:`lzma.LZMADecompressor`, :class:`lzma.LZMACompressor`, -:class:`bz2.BZ2Compressor`, and :class:`bz2.BZ2Decompressor` objects -multiple times. diff --git a/Misc/NEWS.d/next/Library/2022-09-05-12-17-34.gh-issue-88233.gff9qJ.rst b/Misc/NEWS.d/next/Library/2022-09-05-12-17-34.gh-issue-88233.gff9qJ.rst deleted file mode 100644 index 806f7011edc3986..000000000000000 --- a/Misc/NEWS.d/next/Library/2022-09-05-12-17-34.gh-issue-88233.gff9qJ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Correctly preserve "extra" fields in ``zipfile`` regardless of their -ordering relative to a zip64 "extra." diff --git a/Misc/NEWS.d/next/Library/2022-10-22-09-26-43.gh-issue-96764.Dh9Y5L.rst b/Misc/NEWS.d/next/Library/2022-10-22-09-26-43.gh-issue-96764.Dh9Y5L.rst deleted file mode 100644 index a0174291cbc3110..000000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-22-09-26-43.gh-issue-96764.Dh9Y5L.rst +++ /dev/null @@ -1 +0,0 @@ -:func:`asyncio.wait_for` now uses :func:`asyncio.timeout` as its underlying implementation. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2023-01-02-22-41-44.gh-issue-99138.17hp9U.rst b/Misc/NEWS.d/next/Library/2023-01-02-22-41-44.gh-issue-99138.17hp9U.rst deleted file mode 100644 index 3dd4646f40e1e54..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-01-02-22-41-44.gh-issue-99138.17hp9U.rst +++ /dev/null @@ -1 +0,0 @@ -Apply :pep:`687` to :mod:`zoneinfo`. Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2023-01-06-21-14-41.gh-issue-100809.I697UT.rst b/Misc/NEWS.d/next/Library/2023-01-06-21-14-41.gh-issue-100809.I697UT.rst deleted file mode 100644 index 54082de88ccf4ab..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-01-06-21-14-41.gh-issue-100809.I697UT.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix handling of drive-relative paths (like 'C:' and 'C:foo') in -:meth:`pathlib.Path.absolute`. This method now uses the OS API -to retrieve the correct current working directory for the drive. diff --git a/Misc/NEWS.d/next/Library/2023-01-25-00-14-52.gh-issue-101277.FceHX7.rst b/Misc/NEWS.d/next/Library/2023-01-25-00-14-52.gh-issue-101277.FceHX7.rst deleted file mode 100644 index e09c0e09fb388fe..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-01-25-00-14-52.gh-issue-101277.FceHX7.rst +++ /dev/null @@ -1,2 +0,0 @@ -Remove global state from :mod:`itertools` module (:pep:`687`). Patches by -Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2023-01-27-02-53-50.gh-issue-101360.bPB7SL.rst b/Misc/NEWS.d/next/Library/2023-01-27-02-53-50.gh-issue-101360.bPB7SL.rst deleted file mode 100644 index 4cfb136c5db8530..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-01-27-02-53-50.gh-issue-101360.bPB7SL.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix anchor matching in :meth:`pathlib.PureWindowsPath.match`. Path and -pattern anchors are now matched with :mod:`fnmatch`, just like other path -parts. This allows patterns such as ``"*:/Users/*"`` to be matched. diff --git a/Misc/NEWS.d/next/Library/2023-02-01-10-42-16.gh-issue-63301.XNxSFh.rst b/Misc/NEWS.d/next/Library/2023-02-01-10-42-16.gh-issue-63301.XNxSFh.rst deleted file mode 100644 index e00e71fb8554f35..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-01-10-42-16.gh-issue-63301.XNxSFh.rst +++ /dev/null @@ -1 +0,0 @@ -Set exit code when :mod:`tabnanny` CLI exits on error. diff --git a/Misc/NEWS.d/next/Library/2023-02-04-16-35-46.gh-issue-101561.Xo6pIZ.rst b/Misc/NEWS.d/next/Library/2023-02-04-16-35-46.gh-issue-101561.Xo6pIZ.rst deleted file mode 100644 index 2f6a4153062e5a1..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-04-16-35-46.gh-issue-101561.Xo6pIZ.rst +++ /dev/null @@ -1 +0,0 @@ -Add a new decorator :func:`typing.override`. See :pep:`698` for details. Patch by Steven Troxler. diff --git a/Misc/NEWS.d/next/Library/2023-02-05-21-40-15.gh-issue-85984.Kfzbb2.rst b/Misc/NEWS.d/next/Library/2023-02-05-21-40-15.gh-issue-85984.Kfzbb2.rst deleted file mode 100644 index c91829f2c739afb..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-05-21-40-15.gh-issue-85984.Kfzbb2.rst +++ /dev/null @@ -1,4 +0,0 @@ -Refactored the implementation of :func:`pty.fork` to use :func:`os.login_tty`. - -A :exc:`DeprecationWarning` is now raised by ``pty.master_open()`` and ``pty.slave_open()``. They were -undocumented and deprecated long long ago in the docstring in favor of :func:`pty.openpty`. diff --git a/Misc/NEWS.d/next/Library/2023-02-07-20-46-08.gh-issue-101362.2ckZ6R.rst b/Misc/NEWS.d/next/Library/2023-02-07-20-46-08.gh-issue-101362.2ckZ6R.rst deleted file mode 100644 index 8421466cdbb3c9f..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-07-20-46-08.gh-issue-101362.2ckZ6R.rst +++ /dev/null @@ -1,2 +0,0 @@ -Speed up :class:`pathlib.Path` construction by running the path flavour -compatibility check only when pathlib is imported. diff --git a/Misc/NEWS.d/next/Library/2023-02-07-21-16-41.gh-issue-101362.KMQllM.rst b/Misc/NEWS.d/next/Library/2023-02-07-21-16-41.gh-issue-101362.KMQllM.rst deleted file mode 100644 index af4ee9ad904868e..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-07-21-16-41.gh-issue-101362.KMQllM.rst +++ /dev/null @@ -1,2 +0,0 @@ -Speed up :class:`pathlib.PurePath` construction by calling -:func:`os.path.join` only when two or more arguments are given. diff --git a/Misc/NEWS.d/next/Library/2023-02-07-22-20-32.gh-issue-101362.Jlk6mt.rst b/Misc/NEWS.d/next/Library/2023-02-07-22-20-32.gh-issue-101362.Jlk6mt.rst deleted file mode 100644 index c05f92ae699de96..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-07-22-20-32.gh-issue-101362.Jlk6mt.rst +++ /dev/null @@ -1,4 +0,0 @@ -Speed up :class:`pathlib.PurePath` construction by handling arguments more -uniformly. When a :class:`pathlib.Path` argument is supplied, -we use its string representation rather than joining its parts -with :func:`os.path.join`. diff --git a/Misc/NEWS.d/next/Library/2023-02-07-22-21-46.gh-issue-101446.-c0FdK.rst b/Misc/NEWS.d/next/Library/2023-02-07-22-21-46.gh-issue-101446.-c0FdK.rst deleted file mode 100644 index ddf897b71bb1d14..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-07-22-21-46.gh-issue-101446.-c0FdK.rst +++ /dev/null @@ -1,2 +0,0 @@ -Change repr of :class:`collections.OrderedDict` to use regular dictionary -formating instead of pairs of keys and values. diff --git a/Misc/NEWS.d/next/Library/2023-02-08-18-20-58.gh-issue-101693.4_LPXj.rst b/Misc/NEWS.d/next/Library/2023-02-08-18-20-58.gh-issue-101693.4_LPXj.rst deleted file mode 100644 index e436054b15b6576..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-08-18-20-58.gh-issue-101693.4_LPXj.rst +++ /dev/null @@ -1,6 +0,0 @@ -In :meth:`sqlite3.Cursor.execute`, :exc:`DeprecationWarning` is now emitted -when :ref:`named placeholders ` are used together with -parameters supplied as a :term:`sequence` instead of as a :class:`dict`. -Starting from Python 3.14, using named placeholders with parameters supplied -as a sequence will raise a :exc:`~sqlite3.ProgrammingError`. -Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2023-02-10-11-59-13.gh-issue-101773.J_kI7y.rst b/Misc/NEWS.d/next/Library/2023-02-10-11-59-13.gh-issue-101773.J_kI7y.rst deleted file mode 100644 index b577d93d28c2ae9..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-10-11-59-13.gh-issue-101773.J_kI7y.rst +++ /dev/null @@ -1,2 +0,0 @@ -Optimize :class:`fractions.Fraction` for small components. The private argument -``_normalize`` of the :class:`fractions.Fraction` constructor has been removed. diff --git a/Misc/NEWS.d/next/Library/2023-02-10-16-02-29.gh-issue-101517.r7S2u8.rst b/Misc/NEWS.d/next/Library/2023-02-10-16-02-29.gh-issue-101517.r7S2u8.rst deleted file mode 100644 index a5f6bdfa5ac2f0e..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-10-16-02-29.gh-issue-101517.r7S2u8.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed bug where :mod:`bdb` looks up the source line with :mod:`linecache` with a ``lineno=None``, which causes it to fail with an unhandled exception. diff --git a/Misc/NEWS.d/next/Library/2023-02-11-13-23-29.gh-issue-97786.QjvQ1B.rst b/Misc/NEWS.d/next/Library/2023-02-11-13-23-29.gh-issue-97786.QjvQ1B.rst deleted file mode 100644 index df194b67590d67d..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-11-13-23-29.gh-issue-97786.QjvQ1B.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix potential undefined behaviour in corner cases of floating-point-to-time -conversions. diff --git a/Misc/NEWS.d/next/Library/2023-02-13-12-55-48.gh-issue-87634.q-SBhJ.rst b/Misc/NEWS.d/next/Library/2023-02-13-12-55-48.gh-issue-87634.q-SBhJ.rst deleted file mode 100644 index a17927500bd9a54..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-13-12-55-48.gh-issue-87634.q-SBhJ.rst +++ /dev/null @@ -1 +0,0 @@ -Remove locking behavior from :func:`functools.cached_property`. diff --git a/Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst b/Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst deleted file mode 100644 index d586779b3a8a365..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst +++ /dev/null @@ -1,3 +0,0 @@ -Callable iterators no longer raise :class:`SystemError` when the -callable object exhausts the iterator but forgets to either return a -sentinel value or raise :class:`StopIteration`. diff --git a/Misc/NEWS.d/next/Library/2023-02-15-01-54-06.gh-issue-99108.rjTSic.rst b/Misc/NEWS.d/next/Library/2023-02-15-01-54-06.gh-issue-99108.rjTSic.rst deleted file mode 100644 index 1612c89c0ea6bee..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-15-01-54-06.gh-issue-99108.rjTSic.rst +++ /dev/null @@ -1,3 +0,0 @@ -The built-in extension modules for :mod:`hashlib` SHA2 algorithms, used when -OpenSSL does not provide them, now live in a single internal ``_sha2`` module -instead of separate ``_sha256`` and ``_sha512`` modules. diff --git a/Misc/NEWS.d/next/Library/2023-02-17-18-44-27.gh-issue-101997.A6_blD.rst b/Misc/NEWS.d/next/Library/2023-02-17-18-44-27.gh-issue-101997.A6_blD.rst deleted file mode 100644 index f9dfd46d1ed4304..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-17-18-44-27.gh-issue-101997.A6_blD.rst +++ /dev/null @@ -1 +0,0 @@ -Upgrade pip wheel bundled with ensurepip (pip 23.0.1) diff --git a/Misc/NEWS.d/next/Library/2023-02-17-19-00-58.gh-issue-97930.C_nQjb.rst b/Misc/NEWS.d/next/Library/2023-02-17-19-00-58.gh-issue-97930.C_nQjb.rst deleted file mode 100644 index 967e13f752bcd12..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-17-19-00-58.gh-issue-97930.C_nQjb.rst +++ /dev/null @@ -1,4 +0,0 @@ -Apply changes from `importlib_resources 5.12 -`_, -including fix for ``MultiplexedPath`` to support directories in multiple -namespaces (python/importlib_resources#265). diff --git a/Misc/NEWS.d/next/Library/2023-02-17-20-24-15.gh-issue-101566.FjgWBt.rst b/Misc/NEWS.d/next/Library/2023-02-17-20-24-15.gh-issue-101566.FjgWBt.rst deleted file mode 100644 index 5fc1a0288a82dcb..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-17-20-24-15.gh-issue-101566.FjgWBt.rst +++ /dev/null @@ -1,4 +0,0 @@ -In zipfile, sync Path with `zipp 3.14 -`_, including -fix for extractall on the underlying zipfile after being wrapped in -``Path``. diff --git a/Misc/NEWS.d/next/Library/2023-02-21-07-15-41.gh-issue-101936.QVOxHH.rst b/Misc/NEWS.d/next/Library/2023-02-21-07-15-41.gh-issue-101936.QVOxHH.rst deleted file mode 100644 index 55841da44b11461..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-21-07-15-41.gh-issue-101936.QVOxHH.rst +++ /dev/null @@ -1,2 +0,0 @@ -The default value of ``fp`` becomes :class:`io.BytesIO` if :exc:`~urllib.error.HTTPError` -is initialized without a designated ``fp`` parameter. Patch by Long Vo. diff --git a/Misc/NEWS.d/next/Library/2023-02-21-10-05-33.gh-issue-101961.7e56jh.rst b/Misc/NEWS.d/next/Library/2023-02-21-10-05-33.gh-issue-101961.7e56jh.rst deleted file mode 100644 index a3d4119e7cbdceb..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-21-10-05-33.gh-issue-101961.7e56jh.rst +++ /dev/null @@ -1,2 +0,0 @@ -For the binary mode, :func:`fileinput.hookcompressed` doesn't set the ``encoding`` value -even if the value is ``None``. Patch by Gihwan Kim. diff --git a/Misc/NEWS.d/next/Library/2023-02-23-15-06-01.gh-issue-102179.P6KQ4c.rst b/Misc/NEWS.d/next/Library/2023-02-23-15-06-01.gh-issue-102179.P6KQ4c.rst deleted file mode 100644 index f77493e267ac7eb..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-23-15-06-01.gh-issue-102179.P6KQ4c.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :func:`os.dup2` error message for negative fds. diff --git a/Misc/NEWS.d/next/Library/2023-02-23-20-39-52.gh-issue-81652.Vxz0Mr.rst b/Misc/NEWS.d/next/Library/2023-02-23-20-39-52.gh-issue-81652.Vxz0Mr.rst deleted file mode 100644 index 48acce1d863ea60..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-23-20-39-52.gh-issue-81652.Vxz0Mr.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add :data:`mmap.MAP_ALIGNED_SUPER` FreeBSD and :data:`mmap.MAP_CONCEAL` -OpenBSD constants to :mod:`mmap`. Patch by Yeojin Kim. diff --git a/Misc/NEWS.d/next/Library/2023-02-26-12-37-17.gh-issue-91038.S4rFH_.rst b/Misc/NEWS.d/next/Library/2023-02-26-12-37-17.gh-issue-91038.S4rFH_.rst deleted file mode 100644 index 2667ff120fd402e..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-26-12-37-17.gh-issue-91038.S4rFH_.rst +++ /dev/null @@ -1 +0,0 @@ -:meth:`platform.platform` now has boolean default arguments. diff --git a/Misc/NEWS.d/next/Library/2023-02-28-09-52-25.gh-issue-101979.or3hXV.rst b/Misc/NEWS.d/next/Library/2023-02-28-09-52-25.gh-issue-101979.or3hXV.rst deleted file mode 100644 index 1efe72439b3a4ad..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-28-09-52-25.gh-issue-101979.or3hXV.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a bug where parentheses in the ``metavar`` argument to :meth:`argparse.ArgumentParser.add_argument` were -dropped. Patch by Yeojin Kim. diff --git a/Misc/NEWS.d/next/Library/2023-03-04-14-46-47.gh-issue-102302.-b_s6Z.rst b/Misc/NEWS.d/next/Library/2023-03-04-14-46-47.gh-issue-102302.-b_s6Z.rst deleted file mode 100644 index aaf4e62069ca248..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-04-14-46-47.gh-issue-102302.-b_s6Z.rst +++ /dev/null @@ -1 +0,0 @@ -Micro-optimise hashing of :class:`inspect.Parameter`, reducing the time it takes to hash an instance by around 40%. diff --git a/Misc/NEWS.d/next/Security/2023-01-24-16-12-00.gh-issue-101283.9tqu39.rst b/Misc/NEWS.d/next/Security/2023-01-24-16-12-00.gh-issue-101283.9tqu39.rst deleted file mode 100644 index 0efdfa102341856..000000000000000 --- a/Misc/NEWS.d/next/Security/2023-01-24-16-12-00.gh-issue-101283.9tqu39.rst +++ /dev/null @@ -1,3 +0,0 @@ -:class:`subprocess.Popen` now uses a safer approach to find -``cmd.exe`` when launching with ``shell=True``. Patch by Eryk Sun, -based on a patch by Oleg Iarygin. diff --git a/Misc/NEWS.d/next/Security/2023-02-08-12-57-35.gh-issue-99108.6tnmhA.rst b/Misc/NEWS.d/next/Security/2023-02-08-12-57-35.gh-issue-99108.6tnmhA.rst deleted file mode 100644 index 6a7a309dad5d8f9..000000000000000 --- a/Misc/NEWS.d/next/Security/2023-02-08-12-57-35.gh-issue-99108.6tnmhA.rst +++ /dev/null @@ -1,4 +0,0 @@ -Replace the builtin :mod:`hashlib` implementations of SHA2-384 and SHA2-512 -originally from LibTomCrypt with formally verified, side-channel resistant -code from the `HACL* `_ project. -The builtins remain a fallback only used when OpenSSL does not provide them. diff --git a/Misc/NEWS.d/next/Security/2023-02-08-22-03-04.gh-issue-101727.9P5eZz.rst b/Misc/NEWS.d/next/Security/2023-02-08-22-03-04.gh-issue-101727.9P5eZz.rst deleted file mode 100644 index 43acc82063fd7ae..000000000000000 --- a/Misc/NEWS.d/next/Security/2023-02-08-22-03-04.gh-issue-101727.9P5eZz.rst +++ /dev/null @@ -1,4 +0,0 @@ -Updated the OpenSSL version used in Windows and macOS binary release builds -to 1.1.1t to address CVE-2023-0286, CVE-2022-4303, and CVE-2022-4303 per -`the OpenSSL 2023-02-07 security advisory -`_. diff --git a/Misc/NEWS.d/next/Security/2023-02-17-10-42-48.gh-issue-99108.MKA8-f.rst b/Misc/NEWS.d/next/Security/2023-02-17-10-42-48.gh-issue-99108.MKA8-f.rst deleted file mode 100644 index 723d8a43a09f9ef..000000000000000 --- a/Misc/NEWS.d/next/Security/2023-02-17-10-42-48.gh-issue-99108.MKA8-f.rst +++ /dev/null @@ -1,2 +0,0 @@ -Replace builtin hashlib implementations of MD5 and SHA1 with verified ones -from the HACL* project. diff --git a/Misc/NEWS.d/next/Tests/2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst b/Misc/NEWS.d/next/Tests/2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst deleted file mode 100644 index 9de278919ef2f8a..000000000000000 --- a/Misc/NEWS.d/next/Tests/2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst +++ /dev/null @@ -1,4 +0,0 @@ -``test_tools`` now copies up to 10x less source data to a temporary directory -during the ``freeze`` test by ignoring git metadata and other artifacts. It -also limits its python build parallelism based on os.cpu_count instead of hard -coding it as 8 cores. diff --git a/Misc/NEWS.d/next/Tests/2023-02-11-22-36-10.gh-issue-85984.EVXjT9.rst b/Misc/NEWS.d/next/Tests/2023-02-11-22-36-10.gh-issue-85984.EVXjT9.rst deleted file mode 100644 index 402f99ea6c6ebfb..000000000000000 --- a/Misc/NEWS.d/next/Tests/2023-02-11-22-36-10.gh-issue-85984.EVXjT9.rst +++ /dev/null @@ -1 +0,0 @@ -Utilize new "winsize" functions from termios in pty tests. diff --git a/Misc/NEWS.d/next/Tests/2023-02-18-10-51-02.gh-issue-102019.0797SJ.rst b/Misc/NEWS.d/next/Tests/2023-02-18-10-51-02.gh-issue-102019.0797SJ.rst deleted file mode 100644 index 63e36046d26dfee..000000000000000 --- a/Misc/NEWS.d/next/Tests/2023-02-18-10-51-02.gh-issue-102019.0797SJ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix deadlock on shutdown if ``test_current_{exception,frames}`` fails. Patch -by Jacob Bower. diff --git a/Misc/NEWS.d/next/Windows/2023-01-25-11-33-54.gh-issue-101196.wAX_2g.rst b/Misc/NEWS.d/next/Windows/2023-01-25-11-33-54.gh-issue-101196.wAX_2g.rst deleted file mode 100644 index c61e9b90fb53732..000000000000000 --- a/Misc/NEWS.d/next/Windows/2023-01-25-11-33-54.gh-issue-101196.wAX_2g.rst +++ /dev/null @@ -1,3 +0,0 @@ -The functions ``os.path.isdir``, ``os.path.isfile``, ``os.path.islink`` and -``os.path.exists`` are now 13% to 28% faster on Windows, by making fewer Win32 -API calls. diff --git a/Misc/NEWS.d/next/Windows/2023-02-07-18-22-54.gh-issue-101614.NjVP0n.rst b/Misc/NEWS.d/next/Windows/2023-02-07-18-22-54.gh-issue-101614.NjVP0n.rst deleted file mode 100644 index 8ed0995d78925ba..000000000000000 --- a/Misc/NEWS.d/next/Windows/2023-02-07-18-22-54.gh-issue-101614.NjVP0n.rst +++ /dev/null @@ -1 +0,0 @@ -Correctly handle extensions built against debug binaries that reference ``python3_d.dll``. diff --git a/Misc/NEWS.d/next/Windows/2023-02-09-22-09-27.gh-issue-101759.zFlqSH.rst b/Misc/NEWS.d/next/Windows/2023-02-09-22-09-27.gh-issue-101759.zFlqSH.rst deleted file mode 100644 index 62bcac34397d2e0..000000000000000 --- a/Misc/NEWS.d/next/Windows/2023-02-09-22-09-27.gh-issue-101759.zFlqSH.rst +++ /dev/null @@ -1 +0,0 @@ -Update Windows installer to SQLite 3.40.1. diff --git a/Misc/NEWS.d/next/Windows/2023-02-10-14-26-05.gh-issue-101763.RPaj7r.rst b/Misc/NEWS.d/next/Windows/2023-02-10-14-26-05.gh-issue-101763.RPaj7r.rst deleted file mode 100644 index e7e5a73afeb5320..000000000000000 --- a/Misc/NEWS.d/next/Windows/2023-02-10-14-26-05.gh-issue-101763.RPaj7r.rst +++ /dev/null @@ -1 +0,0 @@ -Updates copy of libffi bundled with Windows installs to 3.4.4. diff --git a/Misc/NEWS.d/next/Windows/2023-02-13-16-32-50.gh-issue-101849.7lm_53.rst b/Misc/NEWS.d/next/Windows/2023-02-13-16-32-50.gh-issue-101849.7lm_53.rst deleted file mode 100644 index 861d4de9f9a650a..000000000000000 --- a/Misc/NEWS.d/next/Windows/2023-02-13-16-32-50.gh-issue-101849.7lm_53.rst +++ /dev/null @@ -1 +0,0 @@ -Ensures installer will correctly upgrade existing ``py.exe`` launcher installs. diff --git a/Misc/NEWS.d/next/Windows/2023-02-13-18-05-49.gh-issue-101881._TnHzN.rst b/Misc/NEWS.d/next/Windows/2023-02-13-18-05-49.gh-issue-101881._TnHzN.rst deleted file mode 100644 index ba58dd4f5cb450e..000000000000000 --- a/Misc/NEWS.d/next/Windows/2023-02-13-18-05-49.gh-issue-101881._TnHzN.rst +++ /dev/null @@ -1 +0,0 @@ -Add support for the os.get_blocking() and os.set_blocking() functions on Windows. diff --git a/Misc/NEWS.d/next/Windows/2023-02-15-11-08-10.gh-issue-101881.fScr3m.rst b/Misc/NEWS.d/next/Windows/2023-02-15-11-08-10.gh-issue-101881.fScr3m.rst deleted file mode 100644 index 099b2c1c07a6659..000000000000000 --- a/Misc/NEWS.d/next/Windows/2023-02-15-11-08-10.gh-issue-101881.fScr3m.rst +++ /dev/null @@ -1 +0,0 @@ -Handle read and write operations on non-blocking pipes properly on Windows. diff --git a/Misc/NEWS.d/next/Windows/2023-03-01-01-36-39.gh-issue-102344.Dgfux4.rst b/Misc/NEWS.d/next/Windows/2023-03-01-01-36-39.gh-issue-102344.Dgfux4.rst deleted file mode 100644 index 4804212be8182c8..000000000000000 --- a/Misc/NEWS.d/next/Windows/2023-03-01-01-36-39.gh-issue-102344.Dgfux4.rst +++ /dev/null @@ -1,2 +0,0 @@ -Implement ``winreg.QueryValue`` using ``QueryValueEx`` and -``winreg.SetValue`` using ``SetValueEx``. Patch by Max Bachmann. diff --git a/Misc/NEWS.d/next/macOS/2023-02-09-22-07-17.gh-issue-101759.B0JP2H.rst b/Misc/NEWS.d/next/macOS/2023-02-09-22-07-17.gh-issue-101759.B0JP2H.rst deleted file mode 100644 index fc53d08bffc4fd4..000000000000000 --- a/Misc/NEWS.d/next/macOS/2023-02-09-22-07-17.gh-issue-101759.B0JP2H.rst +++ /dev/null @@ -1 +0,0 @@ -Update macOS installer to SQLite 3.40.1. diff --git a/README.rst b/README.rst index b1756e20c141ab0..6923b692f6c9717 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.12.0 alpha 5 +This is Python version 3.12.0 alpha 6 ===================================== .. image:: https://github.com/python/cpython/workflows/Tests/badge.svg From 1e703a473343ed198c9a06a876b25d7d69d4bbd0 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Tue, 7 Mar 2023 17:10:58 -0700 Subject: [PATCH 32/97] gh-102381: don't call watcher callback with dead object (#102382) Co-authored-by: T. Wouters --- Doc/c-api/code.rst | 17 ++++++++-- Doc/c-api/dict.rst | 21 +++++++++--- Doc/c-api/function.rst | 17 ++++++++-- Include/cpython/code.h | 13 +++++--- Include/cpython/dictobject.h | 21 +++++++----- Include/cpython/funcobject.h | 16 ++++----- Include/internal/pycore_dict.h | 1 + Lib/test/test_capi/test_watchers.py | 52 +++++++++++++++++++++++++++-- Modules/_testcapi/watchers.c | 13 +++++++- Objects/codeobject.c | 38 ++++++++++++++++++++- Objects/dictobject.c | 34 +++++++++++++++++-- Objects/funcobject.c | 38 ++++++++++++++++++++- 12 files changed, 243 insertions(+), 38 deletions(-) diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 062ef3a1fea93cd..a99de9904c07408 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -172,6 +172,11 @@ bound into a function. before the destruction of *co* takes place, so the prior state of *co* can be inspected. + If *event* is ``PY_CODE_EVENT_DESTROY``, taking a reference in the callback + to the about-to-be-destroyed code object will resurrect it and prevent it + from being freed at this time. When the resurrected object is destroyed + later, any watcher callbacks active at that time will be called again. + Users of this API should not rely on internal runtime implementation details. Such details may include, but are not limited to, the exact order and timing of creation and destruction of code objects. While @@ -179,9 +184,15 @@ bound into a function. (including whether a callback is invoked or not), it does not change the semantics of the Python code being executed. - If the callback returns with an exception set, it must return ``-1``; this - exception will be printed as an unraisable exception using - :c:func:`PyErr_WriteUnraisable`. Otherwise it should return ``0``. + If the callback sets an exception, it must return ``-1``; this exception will + be printed as an unraisable exception using :c:func:`PyErr_WriteUnraisable`. + Otherwise it should return ``0``. + + There may already be a pending exception set on entry to the callback. In + this case, the callback should return ``0`` with the same exception still + set. This means the callback may not call any other API that can set an + exception unless it saves and clears the exception state first, and restores + it before returning. .. versionadded:: 3.12 diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 34106ee6b1f30df..b9f84cea7856440 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -298,13 +298,26 @@ Dictionary Objects dictionary. The callback may inspect but must not modify *dict*; doing so could have - unpredictable effects, including infinite recursion. + unpredictable effects, including infinite recursion. Do not trigger Python + code execution in the callback, as it could modify the dict as a side effect. + + If *event* is ``PyDict_EVENT_DEALLOCATED``, taking a new reference in the + callback to the about-to-be-destroyed dictionary will resurrect it and + prevent it from being freed at this time. When the resurrected object is + destroyed later, any watcher callbacks active at that time will be called + again. Callbacks occur before the notified modification to *dict* takes place, so the prior state of *dict* can be inspected. - If the callback returns with an exception set, it must return ``-1``; this - exception will be printed as an unraisable exception using - :c:func:`PyErr_WriteUnraisable`. Otherwise it should return ``0``. + If the callback sets an exception, it must return ``-1``; this exception will + be printed as an unraisable exception using :c:func:`PyErr_WriteUnraisable`. + Otherwise it should return ``0``. + + There may already be a pending exception set on entry to the callback. In + this case, the callback should return ``0`` with the same exception still + set. This means the callback may not call any other API that can set an + exception unless it saves and clears the exception state first, and restores + it before returning. .. versionadded:: 3.12 diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index bc7569d0add97db..947ed70404081b3 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -173,8 +173,19 @@ There are a few functions specific to Python functions. runtime behavior depending on optimization decisions, it does not change the semantics of the Python code being executed. - If the callback returns with an exception set, it must return ``-1``; this - exception will be printed as an unraisable exception using - :c:func:`PyErr_WriteUnraisable`. Otherwise it should return ``0``. + If *event* is ``PyFunction_EVENT_DESTROY``, Taking a reference in the + callback to the about-to-be-destroyed function will resurrect it, preventing + it from being freed at this time. When the resurrected object is destroyed + later, any watcher callbacks active at that time will be called again. + + If the callback sets an exception, it must return ``-1``; this exception will + be printed as an unraisable exception using :c:func:`PyErr_WriteUnraisable`. + Otherwise it should return ``0``. + + There may already be a pending exception set on entry to the callback. In + this case, the callback should return ``0`` with the same exception still + set. This means the callback may not call any other API that can set an + exception unless it saves and clears the exception state first, and restores + it before returning. .. versionadded:: 3.12 diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 0e4bd8a58c165b3..abcf1250603dfed 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -224,9 +224,14 @@ PyAPI_FUNC(int) PyCode_Addr2Line(PyCodeObject *, int); PyAPI_FUNC(int) PyCode_Addr2Location(PyCodeObject *, int, int *, int *, int *, int *); -typedef enum PyCodeEvent { - PY_CODE_EVENT_CREATE, - PY_CODE_EVENT_DESTROY +#define PY_FOREACH_CODE_EVENT(V) \ + V(CREATE) \ + V(DESTROY) + +typedef enum { + #define PY_DEF_EVENT(op) PY_CODE_EVENT_##op, + PY_FOREACH_CODE_EVENT(PY_DEF_EVENT) + #undef PY_DEF_EVENT } PyCodeEvent; @@ -236,7 +241,7 @@ typedef enum PyCodeEvent { * The callback is invoked with a borrowed reference to co, after it is * created and before it is destroyed. * - * If the callback returns with an exception set, it must return -1. Otherwise + * If the callback sets an exception, it must return -1. Otherwise * it should return 0. */ typedef int (*PyCode_WatchCallback)( diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index 5001f35654475e6..ddada922020aa4c 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -16,11 +16,11 @@ typedef struct { /* Dictionary version: globally unique, value change each time the dictionary is modified */ -#ifdef Py_BUILD_CORE +#ifdef Py_BUILD_CORE uint64_t ma_version_tag; #else Py_DEPRECATED(3.12) uint64_t ma_version_tag; -#endif +#endif PyDictKeysObject *ma_keys; @@ -90,13 +90,18 @@ PyAPI_FUNC(PyObject *) _PyDictView_Intersect(PyObject* self, PyObject *other); /* Dictionary watchers */ +#define PY_FOREACH_DICT_EVENT(V) \ + V(ADDED) \ + V(MODIFIED) \ + V(DELETED) \ + V(CLONED) \ + V(CLEARED) \ + V(DEALLOCATED) + typedef enum { - PyDict_EVENT_ADDED, - PyDict_EVENT_MODIFIED, - PyDict_EVENT_DELETED, - PyDict_EVENT_CLONED, - PyDict_EVENT_CLEARED, - PyDict_EVENT_DEALLOCATED, + #define PY_DEF_EVENT(EVENT) PyDict_EVENT_##EVENT, + PY_FOREACH_DICT_EVENT(PY_DEF_EVENT) + #undef PY_DEF_EVENT } PyDict_WatchEvent; // Callback to be invoked when a watched dict is cleared, dealloced, or modified. diff --git a/Include/cpython/funcobject.h b/Include/cpython/funcobject.h index 5979febc2e3421e..c716330cc3fbab9 100644 --- a/Include/cpython/funcobject.h +++ b/Include/cpython/funcobject.h @@ -131,17 +131,17 @@ PyAPI_DATA(PyTypeObject) PyStaticMethod_Type; PyAPI_FUNC(PyObject *) PyClassMethod_New(PyObject *); PyAPI_FUNC(PyObject *) PyStaticMethod_New(PyObject *); -#define FOREACH_FUNC_EVENT(V) \ - V(CREATE) \ - V(DESTROY) \ - V(MODIFY_CODE) \ - V(MODIFY_DEFAULTS) \ +#define PY_FOREACH_FUNC_EVENT(V) \ + V(CREATE) \ + V(DESTROY) \ + V(MODIFY_CODE) \ + V(MODIFY_DEFAULTS) \ V(MODIFY_KWDEFAULTS) typedef enum { - #define DEF_EVENT(EVENT) PyFunction_EVENT_##EVENT, - FOREACH_FUNC_EVENT(DEF_EVENT) - #undef DEF_EVENT + #define PY_DEF_EVENT(EVENT) PyFunction_EVENT_##EVENT, + PY_FOREACH_FUNC_EVENT(PY_DEF_EVENT) + #undef PY_DEF_EVENT } PyFunction_WatchEvent; /* diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index c74a3437713039f..1af5e59a677a9a1 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -164,6 +164,7 @@ _PyDict_NotifyEvent(PyDict_WatchEvent event, PyObject *key, PyObject *value) { + assert(Py_REFCNT((PyObject*)mp) > 0); int watcher_bits = mp->ma_version_tag & DICT_VERSION_MASK; if (watcher_bits) { _PyDict_SendEvent(watcher_bits, event, mp, key, value); diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py index 1922614ef60558a..93f6ef752d0663c 100644 --- a/Lib/test/test_capi/test_watchers.py +++ b/Lib/test/test_capi/test_watchers.py @@ -109,10 +109,21 @@ def test_error(self): self.watch(wid, d) with catch_unraisable_exception() as cm: d["foo"] = "bar" - self.assertIs(cm.unraisable.object, d) + self.assertIn( + "PyDict_EVENT_ADDED watcher callback for 0); PyInterpreterState *interp = _PyInterpreterState_GET(); assert(interp->_initialized); uint8_t bits = interp->active_code_watchers; @@ -25,7 +40,21 @@ notify_code_watchers(PyCodeEvent event, PyCodeObject *co) // callback must be non-null if the watcher bit is set assert(cb != NULL); if (cb(event, co) < 0) { - PyErr_WriteUnraisable((PyObject *) co); + // Don't risk resurrecting the object if an unraisablehook keeps + // a reference; pass a string as context. + PyObject *context = NULL; + PyObject *repr = code_repr(co); + if (repr) { + context = PyUnicode_FromFormat( + "%s watcher callback for %U", + code_event_name(event), repr); + Py_DECREF(repr); + } + if (context == NULL) { + context = Py_NewRef(Py_None); + } + PyErr_WriteUnraisable(context); + Py_DECREF(context); } } i++; @@ -1667,7 +1696,14 @@ code_new_impl(PyTypeObject *type, int argcount, int posonlyargcount, static void code_dealloc(PyCodeObject *co) { + assert(Py_REFCNT(co) == 0); + Py_SET_REFCNT(co, 1); notify_code_watchers(PY_CODE_EVENT_DESTROY, co); + if (Py_REFCNT(co) > 1) { + Py_SET_REFCNT(co, Py_REFCNT(co) - 1); + return; + } + Py_SET_REFCNT(co, 0); if (co->co_extra != NULL) { PyInterpreterState *interp = _PyInterpreterState_GET(); diff --git a/Objects/dictobject.c b/Objects/dictobject.c index fc658ca2f4b7f86..e3795e75e3da5ee 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -2308,7 +2308,14 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) static void dict_dealloc(PyDictObject *mp) { + assert(Py_REFCNT(mp) == 0); + Py_SET_REFCNT(mp, 1); _PyDict_NotifyEvent(PyDict_EVENT_DEALLOCATED, mp, NULL, NULL); + if (Py_REFCNT(mp) > 1) { + Py_SET_REFCNT(mp, Py_REFCNT(mp) - 1); + return; + } + Py_SET_REFCNT(mp, 0); PyDictValues *values = mp->ma_values; PyDictKeysObject *keys = mp->ma_keys; Py_ssize_t i, n; @@ -5732,6 +5739,18 @@ PyDict_ClearWatcher(int watcher_id) return 0; } +static const char * +dict_event_name(PyDict_WatchEvent event) { + switch (event) { + #define CASE(op) \ + case PyDict_EVENT_##op: \ + return "PyDict_EVENT_" #op; + PY_FOREACH_DICT_EVENT(CASE) + #undef CASE + } + Py_UNREACHABLE(); +} + void _PyDict_SendEvent(int watcher_bits, PyDict_WatchEvent event, @@ -5744,9 +5763,18 @@ _PyDict_SendEvent(int watcher_bits, if (watcher_bits & 1) { PyDict_WatchCallback cb = interp->dict_state.watchers[i]; if (cb && (cb(event, (PyObject*)mp, key, value) < 0)) { - // some dict modification paths (e.g. PyDict_Clear) can't raise, so we - // can't propagate exceptions from dict watchers. - PyErr_WriteUnraisable((PyObject *)mp); + // We don't want to resurrect the dict by potentially having an + // unraisablehook keep a reference to it, so we don't pass the + // dict as context, just an informative string message. Dict + // repr can call arbitrary code, so we invent a simpler version. + PyObject *context = PyUnicode_FromFormat( + "%s watcher callback for ", + dict_event_name(event), mp); + if (context == NULL) { + context = Py_NewRef(Py_None); + } + PyErr_WriteUnraisable(context); + Py_DECREF(context); } } watcher_bits >>= 1; diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 91a6b3dd40a2324..99048ea41c6c800 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -8,6 +8,20 @@ #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "structmember.h" // PyMemberDef +static PyObject* func_repr(PyFunctionObject *op); + +static const char * +func_event_name(PyFunction_WatchEvent event) { + switch (event) { + #define CASE(op) \ + case PyFunction_EVENT_##op: \ + return "PyFunction_EVENT_" #op; + PY_FOREACH_FUNC_EVENT(CASE) + #undef CASE + } + Py_UNREACHABLE(); +} + static void notify_func_watchers(PyInterpreterState *interp, PyFunction_WatchEvent event, PyFunctionObject *func, PyObject *new_value) @@ -21,7 +35,21 @@ notify_func_watchers(PyInterpreterState *interp, PyFunction_WatchEvent event, // callback must be non-null if the watcher bit is set assert(cb != NULL); if (cb(event, func, new_value) < 0) { - PyErr_WriteUnraisable((PyObject *) func); + // Don't risk resurrecting the func if an unraisablehook keeps a + // reference; pass a string as context. + PyObject *context = NULL; + PyObject *repr = func_repr(func); + if (repr != NULL) { + context = PyUnicode_FromFormat( + "%s watcher callback for %U", + func_event_name(event), repr); + Py_DECREF(repr); + } + if (context == NULL) { + context = Py_NewRef(Py_None); + } + PyErr_WriteUnraisable(context); + Py_DECREF(context); } } i++; @@ -33,6 +61,7 @@ static inline void handle_func_event(PyFunction_WatchEvent event, PyFunctionObject *func, PyObject *new_value) { + assert(Py_REFCNT(func) > 0); PyInterpreterState *interp = _PyInterpreterState_GET(); assert(interp->_initialized); if (interp->active_func_watchers) { @@ -766,7 +795,14 @@ func_clear(PyFunctionObject *op) static void func_dealloc(PyFunctionObject *op) { + assert(Py_REFCNT(op) == 0); + Py_SET_REFCNT(op, 1); handle_func_event(PyFunction_EVENT_DESTROY, op, NULL); + if (Py_REFCNT(op) > 1) { + Py_SET_REFCNT(op, Py_REFCNT(op) - 1); + return; + } + Py_SET_REFCNT(op, 0); _PyObject_GC_UNTRACK(op); if (op->func_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject *) op); From e499680100a9d4fa4b673cbcfebc32212b4f848a Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Wed, 8 Mar 2023 09:09:50 +0530 Subject: [PATCH 33/97] fix typo in async generator code field name `ag_code` (#102448) --- Objects/genobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/genobject.c b/Objects/genobject.c index 4ab6581e12ab3aa..be08a59ece6b7e7 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1549,7 +1549,7 @@ ag_getframe(PyAsyncGenObject *ag, void *Py_UNUSED(ignored)) static PyObject * ag_getcode(PyGenObject *gen, void *Py_UNUSED(ignored)) { - return _gen_getcode(gen, "ag__code"); + return _gen_getcode(gen, "ag_code"); } static PyGetSetDef async_gen_getsetlist[] = { From 02b9a921cb3e6308ad300ee9c398b24998220931 Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Wed, 8 Mar 2023 05:04:38 +0100 Subject: [PATCH 34/97] Post 3.12.0a6 --- Include/patchlevel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index fd16421eace94af..049cdfa30897cac 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -23,7 +23,7 @@ #define PY_RELEASE_SERIAL 6 /* Version as a string */ -#define PY_VERSION "3.12.0a6" +#define PY_VERSION "3.12.0a6+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. From 061325e0d2bbec6ff89d03f527c91dc7bfa14003 Mon Sep 17 00:00:00 2001 From: Marcin Wieczorek Date: Wed, 8 Mar 2023 08:25:28 +0100 Subject: [PATCH 35/97] Fix style in argparse.rst (#101733) --- Doc/library/argparse.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 34b4c61649b99f4..ee68ac58d3de753 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -34,9 +34,9 @@ around an instance of :class:`argparse.ArgumentParser`. It is a container for argument specifications and has options that apply to the parser as whole:: parser = argparse.ArgumentParser( - prog = 'ProgramName', - description = 'What the program does', - epilog = 'Text at the bottom of help') + prog='ProgramName', + description='What the program does', + epilog='Text at the bottom of help') The :meth:`ArgumentParser.add_argument` method attaches individual argument specifications to the parser. It supports positional arguments, options that From 1a84cc007e207f2dd61f86a7fc3d86632fdce72f Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Wed, 8 Mar 2023 13:29:39 +0530 Subject: [PATCH 36/97] GH-102397: Fix segfault from race condition in signal handling (#102399) Co-authored-by: Gregory P. Smith --- Lib/test/test_signal.py | 15 +++++++++++++++ ...2023-03-04-06-48-34.gh-issue-102397.ACJaOf.rst | 2 ++ Modules/signalmodule.c | 4 ++++ 3 files changed, 21 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-04-06-48-34.gh-issue-102397.ACJaOf.rst diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 2562a57ea421ffd..25afd6aabe07519 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -1406,6 +1406,21 @@ def handler(a, b): signal.raise_signal(signal.SIGINT) self.assertTrue(is_ok) + def test__thread_interrupt_main(self): + # See https://github.com/python/cpython/issues/102397 + code = """if 1: + import _thread + class Foo(): + def __del__(self): + _thread.interrupt_main() + + x = Foo() + """ + + rc, out, err = assert_python_ok('-c', code) + self.assertIn(b'OSError: Signal 2 ignored due to race condition', err) + + class PidfdSignalTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-04-06-48-34.gh-issue-102397.ACJaOf.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-04-06-48-34.gh-issue-102397.ACJaOf.rst new file mode 100644 index 000000000000000..db0b3f32c2ec0b6 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-04-06-48-34.gh-issue-102397.ACJaOf.rst @@ -0,0 +1,2 @@ +Fix segfault from race condition in signal handling during garbage collection. +Patch by Kumar Aditya. diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index cd26eca351c0ed5..0e472e1ee4f9ddc 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -148,6 +148,10 @@ get_signal_state(PyObject *module) static inline int compare_handler(PyObject *func, PyObject *dfl_ign_handler) { + // See https://github.com/python/cpython/pull/102399 + if (func == NULL || dfl_ign_handler == NULL) { + return 0; + } assert(PyLong_CheckExact(dfl_ign_handler)); if (!PyLong_CheckExact(func)) { return 0; From 1f557f94c2ee98c2a43bd090a7bf3f39a22ed874 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 8 Mar 2023 11:19:05 +0300 Subject: [PATCH 37/97] gh-101100: Fix sphinx warnings in `zipapp` and `zipfile` modules (#102526) --- Doc/library/zipapp.rst | 2 +- Doc/library/zipfile.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/zipapp.rst b/Doc/library/zipapp.rst index fb40a2b3e964e4d..981020b13cd988f 100644 --- a/Doc/library/zipapp.rst +++ b/Doc/library/zipapp.rst @@ -215,7 +215,7 @@ using the :func:`create_archive` function:: >>> import zipapp >>> zipapp.create_archive('old_archive.pyz', 'new_archive.pyz', '/usr/bin/python3') -To update the file in place, do the replacement in memory using a :class:`BytesIO` +To update the file in place, do the replacement in memory using a :class:`~io.BytesIO` object, and then overwrite the source afterwards. Note that there is a risk when overwriting a file in place that an error will result in the loss of the original file. This code does not protect against such errors, but diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 0195abc3a992c1b..e2a085d6e98e670 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -288,7 +288,7 @@ ZipFile Objects (``ZipExtFile``) is read-only and provides the following methods: :meth:`~io.BufferedIOBase.read`, :meth:`~io.IOBase.readline`, :meth:`~io.IOBase.readlines`, :meth:`~io.IOBase.seek`, - :meth:`~io.IOBase.tell`, :meth:`__iter__`, :meth:`~iterator.__next__`. + :meth:`~io.IOBase.tell`, :meth:`~container.__iter__`, :meth:`~iterator.__next__`. These objects can operate independently of the ZipFile. With ``mode='w'``, a writable file handle is returned, which supports the From 7d801f245e2021d19daff105ce722f22aa844391 Mon Sep 17 00:00:00 2001 From: sblondon Date: Wed, 8 Mar 2023 11:24:39 +0100 Subject: [PATCH 38/97] Remove or update bitbucket links (GH-101963) Since Mercurial removal from bitbucket.org, some links are broken. They are replaced by github.com or webarchive.org links if available. Otherwise, they are removed. Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- Doc/distributing/index.rst | 3 +-- Doc/installing/index.rst | 3 +-- Doc/library/venv.rst | 2 +- Lib/test/test_pathlib.py | 2 +- PC/launcher.c | 3 +-- PC/launcher2.c | 3 +-- 6 files changed, 6 insertions(+), 10 deletions(-) diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst index 2ae2726d4e4b92a..21389adedf9c154 100644 --- a/Doc/distributing/index.rst +++ b/Doc/distributing/index.rst @@ -39,8 +39,7 @@ Key terms developers and documentation authors responsible for the maintenance and evolution of the standard packaging tools and the associated metadata and file format standards. They maintain a variety of tools, documentation - and issue trackers on both `GitHub `__ and - `Bitbucket `__. + and issue trackers on `GitHub `__. * ``distutils`` is the original build and distribution system first added to the Python standard library in 1998. While direct use of ``distutils`` is being phased out, it still laid the foundation for the current packaging diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst index e158bf1c4c0c7f5..5aec5178d48f3df 100644 --- a/Doc/installing/index.rst +++ b/Doc/installing/index.rst @@ -52,8 +52,7 @@ Key terms developers and documentation authors responsible for the maintenance and evolution of the standard packaging tools and the associated metadata and file format standards. They maintain a variety of tools, documentation, - and issue trackers on both `GitHub `__ and - `Bitbucket `__. + and issue trackers on `GitHub `__. * ``distutils`` is the original build and distribution system first added to the Python standard library in 1998. While direct use of ``distutils`` is being phased out, it still laid the foundation for the current packaging diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 2a41096de006b80..8eb0b35eaa12df6 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -478,7 +478,7 @@ subclass which installs setuptools and pip into a created virtual environment:: :param context: The information for the virtual environment creation request being processed. """ - url = 'https://bitbucket.org/pypa/setuptools/downloads/ez_setup.py' + url = "https://bootstrap.pypa.io/ez_setup.py" self.install_script(context, 'setuptools', url) # clear up the setuptools archive which gets downloaded pred = lambda o: o.startswith('setuptools-') and o.endswith('.tar.gz') diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index df9c1f6ba65deb5..f8e2f44d27fc1ef 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -1938,7 +1938,7 @@ def test_resolve_common(self): @os_helper.skip_unless_symlink def test_resolve_dot(self): - # See https://bitbucket.org/pitrou/pathlib/issue/9/pathresolve-fails-on-complex-symlinks + # See http://web.archive.org/web/20200623062557/https://bitbucket.org/pitrou/pathlib/issues/9/ p = self.cls(BASE) self.dirlink('.', join('0')) self.dirlink(os.path.join('0', '0'), join('1')) diff --git a/PC/launcher.c b/PC/launcher.c index da566a180168c5d..0776e57249c427b 100644 --- a/PC/launcher.c +++ b/PC/launcher.c @@ -770,8 +770,7 @@ run_child(wchar_t * cmdline) window, or fetching a message). As this launcher doesn't do this directly, that cursor remains even after the child process does these things. We avoid that by doing a simple post+get message. - See http://bugs.python.org/issue17290 and - https://bitbucket.org/vinay.sajip/pylauncher/issue/20/busy-cursor-for-a-long-time-when-running + See http://bugs.python.org/issue17290 */ MSG msg; diff --git a/PC/launcher2.c b/PC/launcher2.c index beeb2ae46b83f04..932665387f19666 100644 --- a/PC/launcher2.c +++ b/PC/launcher2.c @@ -2473,8 +2473,7 @@ launchEnvironment(const SearchInfo *search, const EnvironmentInfo *launch, wchar window, or fetching a message). As this launcher doesn't do this directly, that cursor remains even after the child process does these things. We avoid that by doing a simple post+get message. - See http://bugs.python.org/issue17290 and - https://bitbucket.org/vinay.sajip/pylauncher/issue/20/busy-cursor-for-a-long-time-when-running + See http://bugs.python.org/issue17290 */ MSG msg; From 401d7a7f009ca2e282b1a0d1b880dc602afd39dc Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 8 Mar 2023 11:45:38 +0000 Subject: [PATCH 39/97] gh-102515: Remove unused imports in the `Lib/` directory (#102516) --- Lib/_pylong.py | 1 - Lib/concurrent/futures/process.py | 2 ++ Lib/dataclasses.py | 1 - Lib/importlib/_abc.py | 1 - Lib/sysconfig.py | 2 +- Lib/test/_test_venv_multiprocessing.py | 1 - Lib/test/fork_wait.py | 2 +- Lib/test/test__xxinterpchannels.py | 1 - Lib/test/test__xxsubinterpreters.py | 2 -- Lib/test/test_asyncgen.py | 1 - Lib/test/test_asyncio/test_ssl.py | 1 - Lib/test/test_asyncio/test_subprocess.py | 1 - Lib/test/test_ctypes/test_callbacks.py | 1 - Lib/test/test_dataclasses.py | 1 - Lib/test/test_enum.py | 2 -- Lib/test/test_grammar.py | 2 -- Lib/test/test_hashlib.py | 1 - Lib/test/test_httplib.py | 2 -- Lib/test/test_imaplib.py | 2 -- Lib/test/test_isinstance.py | 1 - Lib/test/test_minidom.py | 1 - Lib/test/test_peg_generator/test_c_parser.py | 1 - Lib/test/test_shlex.py | 1 - Lib/test/test_sys_setprofile.py | 1 - Lib/test/test_sys_settrace.py | 1 - Lib/test/test_tkinter/test_widgets.py | 3 +-- Lib/test/test_tools/test_sundry.py | 1 - Lib/test/test_traceback.py | 1 - Lib/test/test_ttk/test_widgets.py | 4 +--- Lib/test/test_unicodedata.py | 3 +-- Lib/test/test_unittest/test_loader.py | 1 - Lib/test/test_wmi.py | 1 - Lib/test/test_zipfile64.py | 2 +- 33 files changed, 8 insertions(+), 41 deletions(-) diff --git a/Lib/_pylong.py b/Lib/_pylong.py index d14c1d938363279..936346e187ff699 100644 --- a/Lib/_pylong.py +++ b/Lib/_pylong.py @@ -12,7 +12,6 @@ tricky or non-obvious code is not worth it. For people looking for maximum performance, they should use something like gmpy2.""" -import sys import re import decimal diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index bee162430a6f8e1..3a8637b6fa1b47f 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -49,6 +49,8 @@ from concurrent.futures import _base import queue import multiprocessing as mp +# This import is required to load the multiprocessing.connection submodule +# so that it can be accessed later as `mp.connection` import multiprocessing.connection from multiprocessing.queues import Queue import threading diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 5c0257eba186d15..8bc8594d674bc0d 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -4,7 +4,6 @@ import types import inspect import keyword -import builtins import functools import itertools import abc diff --git a/Lib/importlib/_abc.py b/Lib/importlib/_abc.py index 083205638965215..693b466112638f5 100644 --- a/Lib/importlib/_abc.py +++ b/Lib/importlib/_abc.py @@ -1,7 +1,6 @@ """Subset of importlib.abc used to reduce importlib.util imports.""" from . import _bootstrap import abc -import warnings class Loader(metaclass=abc.ABCMeta): diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index c61100a6da85033..122d441bd19f5e8 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -3,7 +3,7 @@ import os import sys import threading -from os.path import pardir, realpath +from os.path import realpath __all__ = [ 'get_config_h_filename', diff --git a/Lib/test/_test_venv_multiprocessing.py b/Lib/test/_test_venv_multiprocessing.py index 044a0c6cd3f5ca0..ad985dd8d56bb4b 100644 --- a/Lib/test/_test_venv_multiprocessing.py +++ b/Lib/test/_test_venv_multiprocessing.py @@ -1,7 +1,6 @@ import multiprocessing import random import sys -import time def fill_queue(queue, code): queue.put(code) diff --git a/Lib/test/fork_wait.py b/Lib/test/fork_wait.py index c26c7aaaeb43061..8c32895f5e09e5d 100644 --- a/Lib/test/fork_wait.py +++ b/Lib/test/fork_wait.py @@ -9,7 +9,7 @@ active threads survive in the child after a fork(); this is an error. """ -import os, sys, time, unittest +import os, time, unittest import threading from test import support from test.support import threading_helper diff --git a/Lib/test/test__xxinterpchannels.py b/Lib/test/test__xxinterpchannels.py index 03bb5c80b8dac92..69bda89a1688f57 100644 --- a/Lib/test/test__xxinterpchannels.py +++ b/Lib/test/test__xxinterpchannels.py @@ -1,6 +1,5 @@ from collections import namedtuple import contextlib -import os import sys from textwrap import dedent import threading diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 687fcf3b770522a..965967e3f2734b4 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -1,4 +1,3 @@ -from collections import namedtuple import contextlib import itertools import os @@ -6,7 +5,6 @@ import sys from textwrap import dedent import threading -import time import unittest import _testcapi diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 0421efdbf9dac9a..09e4010b0e53d6f 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -2,7 +2,6 @@ import types import unittest import contextlib -import warnings from test.support.import_helper import import_module from test.support import gc_collect, requires_working_socket diff --git a/Lib/test/test_asyncio/test_ssl.py b/Lib/test/test_asyncio/test_ssl.py index aaf3c37101f52a6..e9cc735613fb8e1 100644 --- a/Lib/test/test_asyncio/test_ssl.py +++ b/Lib/test/test_asyncio/test_ssl.py @@ -1,5 +1,4 @@ import asyncio -import asyncio.sslproto import contextlib import gc import logging diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index f1ad10a9903fe86..eba6e2d1f28f3ec 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -1,5 +1,4 @@ import os -import shutil import signal import sys import unittest diff --git a/Lib/test/test_ctypes/test_callbacks.py b/Lib/test/test_ctypes/test_callbacks.py index e8fa3e6f7aca514..b185e388ab1527d 100644 --- a/Lib/test/test_ctypes/test_callbacks.py +++ b/Lib/test/test_ctypes/test_callbacks.py @@ -1,4 +1,3 @@ -import sys import functools import unittest from test import support diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 81a36aa241acf76..589f229f4623594 100644 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -2261,7 +2261,6 @@ def test_base_has_init(self): class B: def __init__(self): self.z = 100 - pass # Make sure that declaring this class doesn't raise an error. # The issue is that we can't override __init__ in our class, diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 0a2e0c14d268af1..2b14590b2c21af5 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -20,7 +20,6 @@ from test import support from test.support import ALWAYS_EQ from test.support import threading_helper -from textwrap import dedent from datetime import timedelta python_version = sys.version_info[:2] @@ -1187,7 +1186,6 @@ class MyEnum(HexInt, enum.Enum): # class SillyInt(HexInt): __qualname__ = 'SillyInt' - pass class MyOtherEnum(SillyInt, enum.Enum): __qualname__ = 'MyOtherEnum' D = 4 diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 5b946020994e31e..ced9000f75f2e57 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -3,7 +3,6 @@ from test.support import check_syntax_error from test.support import import_helper -from test.support.warnings_helper import check_syntax_warning import inspect import unittest import sys @@ -15,7 +14,6 @@ # with import machinery import test.ann_module as ann_module import typing -from collections import ChainMap from test import ann_module2 import test diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 5ead8857943592d..08cb5eb0c2bbab8 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -22,7 +22,6 @@ from test.support import os_helper from test.support import requires_resource from test.support import threading_helper -from test.support import warnings_helper from http.client import HTTPException diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 620a5b19109a8c1..9ff6afcbadec54c 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -8,7 +8,6 @@ import re import socket import threading -import warnings import unittest from unittest import mock @@ -17,7 +16,6 @@ from test import support from test.support import os_helper from test.support import socket_helper -from test.support import warnings_helper support.requires_working_socket(module=True) diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 7626d9572e1e969..60f5b671b1da485 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -13,7 +13,6 @@ from test.support import verbose, run_with_tz, run_with_locale, cpython_only from test.support import hashlib_helper from test.support import threading_helper -from test.support import warnings_helper import unittest from unittest import mock from datetime import datetime, timezone, timedelta @@ -751,7 +750,6 @@ class NonUTF8Server(SimpleIMAPHandler): typ, data = client.login('user', 'pass') self.assertEqual(typ, 'OK') client.enable('UTF8=ACCEPT') - pass @threading_helper.reap_threads def test_enable_UTF8_True_append(self): diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index 2fcf6ebbee7e346..bf9332e40aeaf2e 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -3,7 +3,6 @@ # testing of error conditions uncovered when using extension types. import unittest -import sys import typing from test import support diff --git a/Lib/test/test_minidom.py b/Lib/test/test_minidom.py index 2ca3908bd1caac1..699265ccadc7f9c 100644 --- a/Lib/test/test_minidom.py +++ b/Lib/test/test_minidom.py @@ -6,7 +6,6 @@ from test import support import unittest -import pyexpat import xml.dom.minidom from xml.dom.minidom import parse, Attr, Node, Document, parseString diff --git a/Lib/test/test_peg_generator/test_c_parser.py b/Lib/test/test_peg_generator/test_c_parser.py index 1b3fcbb92f82923..d34ffef0dbc5ecc 100644 --- a/Lib/test/test_peg_generator/test_c_parser.py +++ b/Lib/test/test_peg_generator/test_c_parser.py @@ -24,7 +24,6 @@ generate_parser_c_extension, generate_c_parser_source, ) - from pegen.ast_dump import ast_dump TEST_TEMPLATE = """ diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py index 92598dbbd5f293c..797c91ee7effdfd 100644 --- a/Lib/test/test_shlex.py +++ b/Lib/test/test_shlex.py @@ -3,7 +3,6 @@ import shlex import string import unittest -from unittest import mock # The original test data set was from shellwords, by Hartmut Goebel. diff --git a/Lib/test/test_sys_setprofile.py b/Lib/test/test_sys_setprofile.py index acae433cd0a549d..49e076c77d167ae 100644 --- a/Lib/test/test_sys_setprofile.py +++ b/Lib/test/test_sys_setprofile.py @@ -2,7 +2,6 @@ import pprint import sys import unittest -from test import support class TestGetProfile(unittest.TestCase): diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index a251b2272e95eb0..4907c930e143d50 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -2,7 +2,6 @@ from test import support import unittest -from unittest.mock import MagicMock import sys import difflib import gc diff --git a/Lib/test/test_tkinter/test_widgets.py b/Lib/test/test_tkinter/test_widgets.py index 6fde93cbecc73ff..64c9472706549bb 100644 --- a/Lib/test/test_tkinter/test_widgets.py +++ b/Lib/test/test_tkinter/test_widgets.py @@ -9,8 +9,7 @@ AbstractDefaultRootTest) from test.test_tkinter.widget_tests import ( add_standard_options, - AbstractWidgetTest, StandardOptionsTests, IntegerSizeTests, PixelSizeTests, - setUpModule) + AbstractWidgetTest, StandardOptionsTests, IntegerSizeTests, PixelSizeTests) requires('gui') diff --git a/Lib/test/test_tools/test_sundry.py b/Lib/test/test_tools/test_sundry.py index 81f06763980a325..6a3dc12781b2b6b 100644 --- a/Lib/test/test_tools/test_sundry.py +++ b/Lib/test/test_tools/test_sundry.py @@ -6,7 +6,6 @@ """ import os -import sys import unittest from test.support import import_helper diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 95b1bae4f60850b..92c5a000585855e 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -6,7 +6,6 @@ import sys import types import inspect -import importlib import builtins import unittest import re diff --git a/Lib/test/test_ttk/test_widgets.py b/Lib/test/test_ttk/test_widgets.py index 6f47ccb8e8b3ded..79d65b496abdc64 100644 --- a/Lib/test/test_ttk/test_widgets.py +++ b/Lib/test/test_ttk/test_widgets.py @@ -8,8 +8,7 @@ from test.test_tkinter.support import (AbstractTkTest, tcl_version, get_tk_patchlevel, simulate_mouse_click, AbstractDefaultRootTest) from test.test_tkinter.widget_tests import (add_standard_options, - AbstractWidgetTest, StandardOptionsTests, IntegerSizeTests, PixelSizeTests, - setUpModule) + AbstractWidgetTest, StandardOptionsTests, IntegerSizeTests, PixelSizeTests) requires('gui') @@ -50,7 +49,6 @@ def test_configure_style(self): widget2 = self.create(class_='Foo') self.assertEqual(widget2['class'], 'Foo') # XXX - pass class WidgetTest(AbstractTkTest, unittest.TestCase): diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index 74503c89e559a04..3dc0790ca15b416 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -12,8 +12,7 @@ import unicodedata import unittest from test.support import (open_urlresource, requires_resource, script_helper, - cpython_only, check_disallow_instantiation, - ResourceDenied) + cpython_only, check_disallow_instantiation) class UnicodeMethodsTest(unittest.TestCase): diff --git a/Lib/test/test_unittest/test_loader.py b/Lib/test/test_unittest/test_loader.py index bbdfb247ebaada4..a203145a791b1a0 100644 --- a/Lib/test/test_unittest/test_loader.py +++ b/Lib/test/test_unittest/test_loader.py @@ -1,7 +1,6 @@ import functools import sys import types -import warnings import unittest diff --git a/Lib/test/test_wmi.py b/Lib/test/test_wmi.py index 3f5795952905244..3445702846d8a04 100644 --- a/Lib/test/test_wmi.py +++ b/Lib/test/test_wmi.py @@ -1,7 +1,6 @@ # Test the internal _wmi module on Windows # This is used by the platform module, and potentially others -import sys import unittest from test.support import import_helper, requires_resource diff --git a/Lib/test/test_zipfile64.py b/Lib/test/test_zipfile64.py index be654a8478b04b7..2e1affe02528581 100644 --- a/Lib/test/test_zipfile64.py +++ b/Lib/test/test_zipfile64.py @@ -11,7 +11,7 @@ 'test requires loads of disk-space bytes and a long time to run' ) -import zipfile, os, unittest +import zipfile, unittest import time import sys From b097925858c6975c73e989226cf278cc382c0416 Mon Sep 17 00:00:00 2001 From: JosephSBoyle <48555120+JosephSBoyle@users.noreply.github.com> Date: Wed, 8 Mar 2023 13:58:14 +0000 Subject: [PATCH 40/97] gh-102507 Remove invisible pagebreak characters (#102531) Co-authored-by: AlexWaygood --- Lib/email/__init__.py | 1 - Lib/email/base64mime.py | 4 ---- Lib/email/charset.py | 5 ----- Lib/email/encoders.py | 4 ---- Lib/email/feedparser.py | 2 -- Lib/email/generator.py | 5 +---- Lib/email/header.py | 5 ----- Lib/email/iterators.py | 3 --- Lib/email/mime/base.py | 1 - Lib/email/mime/message.py | 1 - Lib/email/mime/multipart.py | 1 - Lib/email/mime/nonmultipart.py | 1 - Lib/email/mime/text.py | 1 - Lib/email/parser.py | 3 +-- Modules/_io/bufferedio.c | 3 --- Tools/i18n/pygettext.py | 11 ++++------- 16 files changed, 6 insertions(+), 45 deletions(-) diff --git a/Lib/email/__init__.py b/Lib/email/__init__.py index fae872439edc66d..9fa477830041859 100644 --- a/Lib/email/__init__.py +++ b/Lib/email/__init__.py @@ -25,7 +25,6 @@ ] - # Some convenience routines. Don't import Parser and Message as side-effects # of importing email since those cascadingly import most of the rest of the # email package. diff --git a/Lib/email/base64mime.py b/Lib/email/base64mime.py index a7cc37365c6f9a1..4cdf22666e30161 100644 --- a/Lib/email/base64mime.py +++ b/Lib/email/base64mime.py @@ -45,7 +45,6 @@ MISC_LEN = 7 - # Helpers def header_length(bytearray): """Return the length of s when it is encoded with base64.""" @@ -57,7 +56,6 @@ def header_length(bytearray): return n - def header_encode(header_bytes, charset='iso-8859-1'): """Encode a single header line with Base64 encoding in a given charset. @@ -72,7 +70,6 @@ def header_encode(header_bytes, charset='iso-8859-1'): return '=?%s?b?%s?=' % (charset, encoded) - def body_encode(s, maxlinelen=76, eol=NL): r"""Encode a string with base64. @@ -98,7 +95,6 @@ def body_encode(s, maxlinelen=76, eol=NL): return EMPTYSTRING.join(encvec) - def decode(string): """Decode a raw base64 string, returning a bytes object. diff --git a/Lib/email/charset.py b/Lib/email/charset.py index 791b6584b247572..9af269442fb8afb 100644 --- a/Lib/email/charset.py +++ b/Lib/email/charset.py @@ -18,7 +18,6 @@ from email.encoders import encode_7or8bit - # Flags for types of header encodings QP = 1 # Quoted-Printable BASE64 = 2 # Base64 @@ -32,7 +31,6 @@ EMPTYSTRING = '' - # Defaults CHARSETS = { # input header enc body enc output conv @@ -104,7 +102,6 @@ } - # Convenience functions for extending the above mappings def add_charset(charset, header_enc=None, body_enc=None, output_charset=None): """Add character set properties to the global registry. @@ -153,7 +150,6 @@ def add_codec(charset, codecname): CODEC_MAP[charset] = codecname - # Convenience function for encoding strings, taking into account # that they might be unknown-8bit (ie: have surrogate-escaped bytes) def _encode(string, codec): @@ -163,7 +159,6 @@ def _encode(string, codec): return string.encode(codec) - class Charset: """Map character sets to their email properties. diff --git a/Lib/email/encoders.py b/Lib/email/encoders.py index 0a66acb6240bd78..17bd1ab7b19f325 100644 --- a/Lib/email/encoders.py +++ b/Lib/email/encoders.py @@ -16,7 +16,6 @@ from quopri import encodestring as _encodestring - def _qencode(s): enc = _encodestring(s, quotetabs=True) # Must encode spaces, which quopri.encodestring() doesn't do @@ -34,7 +33,6 @@ def encode_base64(msg): msg['Content-Transfer-Encoding'] = 'base64' - def encode_quopri(msg): """Encode the message's payload in quoted-printable. @@ -46,7 +44,6 @@ def encode_quopri(msg): msg['Content-Transfer-Encoding'] = 'quoted-printable' - def encode_7or8bit(msg): """Set the Content-Transfer-Encoding header to 7bit or 8bit.""" orig = msg.get_payload(decode=True) @@ -64,6 +61,5 @@ def encode_7or8bit(msg): msg['Content-Transfer-Encoding'] = '7bit' - def encode_noop(msg): """Do nothing.""" diff --git a/Lib/email/feedparser.py b/Lib/email/feedparser.py index 97d3f5144d606ff..6bc4e0c4e598957 100644 --- a/Lib/email/feedparser.py +++ b/Lib/email/feedparser.py @@ -41,7 +41,6 @@ NeedMoreData = object() - class BufferedSubFile(object): """A file-ish object that can have new data loaded into it. @@ -132,7 +131,6 @@ def __next__(self): return line - class FeedParser: """A feed-style parser of email.""" diff --git a/Lib/email/generator.py b/Lib/email/generator.py index 885e6ba98540a7d..7ccbe10eb768567 100644 --- a/Lib/email/generator.py +++ b/Lib/email/generator.py @@ -22,7 +22,6 @@ fcre = re.compile(r'^From ', re.MULTILINE) - class Generator: """Generates output from a Message object tree. @@ -392,7 +391,7 @@ def _make_boundary(cls, text=None): def _compile_re(cls, s, flags): return re.compile(s, flags) - + class BytesGenerator(Generator): """Generates a bytes version of a Message object tree. @@ -443,7 +442,6 @@ def _compile_re(cls, s, flags): return re.compile(s.encode('ascii'), flags) - _FMT = '[Non-text (%(type)s) part of message omitted, filename %(filename)s]' class DecodedGenerator(Generator): @@ -503,7 +501,6 @@ def _dispatch(self, msg): }, file=self) - # Helper used by Generator._make_boundary _width = len(repr(sys.maxsize-1)) _fmt = '%%0%dd' % _width diff --git a/Lib/email/header.py b/Lib/email/header.py index 4ab0032bc661234..984851a7d9a679b 100644 --- a/Lib/email/header.py +++ b/Lib/email/header.py @@ -52,12 +52,10 @@ _embedded_header = re.compile(r'\n[^ \t]+:') - # Helpers _max_append = email.quoprimime._max_append - def decode_header(header): """Decode a message header value without converting charset. @@ -152,7 +150,6 @@ def decode_header(header): return collapsed - def make_header(decoded_seq, maxlinelen=None, header_name=None, continuation_ws=' '): """Create a Header from a sequence of pairs as returned by decode_header() @@ -175,7 +172,6 @@ def make_header(decoded_seq, maxlinelen=None, header_name=None, return h - class Header: def __init__(self, s=None, charset=None, maxlinelen=None, header_name=None, @@ -409,7 +405,6 @@ def _normalize(self): self._chunks = chunks - class _ValueFormatter: def __init__(self, headerlen, maxlen, continuation_ws, splitchars): self._maxlen = maxlen diff --git a/Lib/email/iterators.py b/Lib/email/iterators.py index b5502ee975266ba..3410935e38f4760 100644 --- a/Lib/email/iterators.py +++ b/Lib/email/iterators.py @@ -15,7 +15,6 @@ from io import StringIO - # This function will become a method of the Message class def walk(self): """Walk over the message tree, yielding each subpart. @@ -29,7 +28,6 @@ def walk(self): yield from subpart.walk() - # These two functions are imported into the Iterators.py interface module. def body_line_iterator(msg, decode=False): """Iterate over the parts, returning string payloads line-by-line. @@ -55,7 +53,6 @@ def typed_subpart_iterator(msg, maintype='text', subtype=None): yield subpart - def _structure(msg, fp=None, level=0, include_default=False): """A handy debugging aid""" if fp is None: diff --git a/Lib/email/mime/base.py b/Lib/email/mime/base.py index 1a3f9b51f6c0456..f601f621cec3933 100644 --- a/Lib/email/mime/base.py +++ b/Lib/email/mime/base.py @@ -11,7 +11,6 @@ from email import message - class MIMEBase(message.Message): """Base class for MIME specializations.""" diff --git a/Lib/email/mime/message.py b/Lib/email/mime/message.py index 07e4f2d11961516..61836b5a7861fca 100644 --- a/Lib/email/mime/message.py +++ b/Lib/email/mime/message.py @@ -10,7 +10,6 @@ from email.mime.nonmultipart import MIMENonMultipart - class MIMEMessage(MIMENonMultipart): """Class representing message/* MIME documents.""" diff --git a/Lib/email/mime/multipart.py b/Lib/email/mime/multipart.py index 2d3f288810dd919..94d81c771a474eb 100644 --- a/Lib/email/mime/multipart.py +++ b/Lib/email/mime/multipart.py @@ -9,7 +9,6 @@ from email.mime.base import MIMEBase - class MIMEMultipart(MIMEBase): """Base class for MIME multipart/* type messages.""" diff --git a/Lib/email/mime/nonmultipart.py b/Lib/email/mime/nonmultipart.py index e1f51968b59eb15..a41386eb148c0c8 100644 --- a/Lib/email/mime/nonmultipart.py +++ b/Lib/email/mime/nonmultipart.py @@ -10,7 +10,6 @@ from email.mime.base import MIMEBase - class MIMENonMultipart(MIMEBase): """Base class for MIME non-multipart type messages.""" diff --git a/Lib/email/mime/text.py b/Lib/email/mime/text.py index 35b442383002b22..dfe53c426b2ac40 100644 --- a/Lib/email/mime/text.py +++ b/Lib/email/mime/text.py @@ -10,7 +10,6 @@ from email.mime.nonmultipart import MIMENonMultipart - class MIMEText(MIMENonMultipart): """Class for generating text/* type MIME documents.""" diff --git a/Lib/email/parser.py b/Lib/email/parser.py index e94d455baa52622..06d99b17f2f9c49 100644 --- a/Lib/email/parser.py +++ b/Lib/email/parser.py @@ -64,7 +64,6 @@ def parsestr(self, text, headersonly=False): return self.parse(StringIO(text), headersonly=headersonly) - class HeaderParser(Parser): def parse(self, fp, headersonly=True): return Parser.parse(self, fp, True) @@ -72,7 +71,7 @@ def parse(self, fp, headersonly=True): def parsestr(self, text, headersonly=True): return Parser.parsestr(self, text, True) - + class BytesParser: def __init__(self, *args, **kw): diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 960026707fc5ed7..2c71768be978702 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -1742,7 +1742,6 @@ _bufferedreader_peek_unlocked(buffered *self) self->pos = 0; return PyBytes_FromStringAndSize(self->buffer, r); } - /* @@ -2052,7 +2051,6 @@ _io_BufferedWriter_write_impl(buffered *self, Py_buffer *buffer) LEAVE_BUFFERED(self) return res; } - /* @@ -2266,7 +2264,6 @@ bufferedrwpair_closed_get(rwpair *self, void *context) } return PyObject_GetAttr((PyObject *) self->writer, &_Py_ID(closed)); } - /* diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index 7ada79105db1cad..3a0b27ba420e7a7 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -174,7 +174,6 @@ EMPTYSTRING = '' - # The normal pot-file header. msgmerge and Emacs's po-mode work better if it's # there. pot_header = _('''\ @@ -196,7 +195,7 @@ ''') - + def usage(code, msg=''): print(__doc__ % globals(), file=sys.stderr) if msg: @@ -204,7 +203,6 @@ def usage(code, msg=''): sys.exit(code) - def make_escapes(pass_nonascii): global escapes, escape if pass_nonascii: @@ -258,7 +256,7 @@ def normalize(s, encoding): s = '""\n"' + lineterm.join(lines) + '"' return s - + def containsAny(str, set): """Check whether 'str' contains ANY of the chars in 'set'""" return 1 in [c in str for c in set] @@ -307,7 +305,7 @@ def getFilesForName(name): return [] - + class TokenEater: def __init__(self, options): self.__options = options @@ -515,7 +513,6 @@ def write(self, fp): print('msgstr ""\n', file=fp) - def main(): global default_keywords try: @@ -675,7 +672,7 @@ class Options: if closep: fp.close() - + if __name__ == '__main__': main() # some more test strings From 11a2c6ce516b24b2435cb627742a6c4df92d411c Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 8 Mar 2023 17:03:18 +0000 Subject: [PATCH 41/97] gh-102192: Replace PyErr_Fetch/Restore etc by more efficient alternatives (in Objects/) (#102218) --- Objects/dictobject.c | 2 +- Objects/exceptions.c | 39 +++++++++---------------- Objects/frameobject.c | 5 ++-- Objects/genobject.c | 64 +++++++++++------------------------------ Objects/object.c | 25 +++++++--------- Objects/odictobject.c | 5 ++-- Objects/typeobject.c | 10 +++---- Objects/weakrefobject.c | 7 ++--- 8 files changed, 51 insertions(+), 106 deletions(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index e3795e75e3da5ee..75c92172a91778c 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -119,7 +119,7 @@ As a consequence of this, split keys have a maximum size of 16. #include "pycore_dict.h" // PyDictKeysObject #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() #include "pycore_object.h" // _PyObject_GC_TRACK() -#include "pycore_pyerrors.h" // _PyErr_Fetch() +#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyThreadState_GET() #include "stringlib/eq.h" // unicode_eq() diff --git a/Objects/exceptions.c b/Objects/exceptions.c index a473cbdfeda7fc1..c6fb6a3f19b2d06 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -3781,16 +3781,13 @@ PyObject * _PyErr_TrySetFromCause(const char *format, ...) { PyObject* msg_prefix; - PyObject *exc, *val, *tb; - PyTypeObject *caught_type; PyObject *instance_args; Py_ssize_t num_args, caught_type_size, base_exc_size; - PyObject *new_exc, *new_val, *new_tb; va_list vargs; int same_basic_size; - PyErr_Fetch(&exc, &val, &tb); - caught_type = (PyTypeObject *)exc; + PyObject *exc = PyErr_GetRaisedException(); + PyTypeObject *caught_type = Py_TYPE(exc); /* Ensure type info indicates no extra state is stored at the C level * and that the type can be reinstantiated using PyErr_Format */ @@ -3810,31 +3807,30 @@ _PyErr_TrySetFromCause(const char *format, ...) * more state than just the exception type. Accordingly, we just * leave it alone. */ - PyErr_Restore(exc, val, tb); + PyErr_SetRaisedException(exc); return NULL; } /* Check the args are empty or contain a single string */ - PyErr_NormalizeException(&exc, &val, &tb); - instance_args = ((PyBaseExceptionObject *)val)->args; + instance_args = ((PyBaseExceptionObject *)exc)->args; num_args = PyTuple_GET_SIZE(instance_args); if (num_args > 1 || (num_args == 1 && !PyUnicode_CheckExact(PyTuple_GET_ITEM(instance_args, 0)))) { /* More than 1 arg, or the one arg we do have isn't a string */ - PyErr_Restore(exc, val, tb); + PyErr_SetRaisedException(exc); return NULL; } /* Ensure the instance dict is also empty */ - if (!_PyObject_IsInstanceDictEmpty(val)) { + if (!_PyObject_IsInstanceDictEmpty(exc)) { /* While we could potentially copy a non-empty instance dictionary * to the replacement exception, for now we take the more * conservative path of leaving exceptions with attributes set * alone. */ - PyErr_Restore(exc, val, tb); + PyErr_SetRaisedException(exc); return NULL; } @@ -3847,28 +3843,19 @@ _PyErr_TrySetFromCause(const char *format, ...) * types as well, but that's quite a bit trickier due to the extra * state potentially stored on OSError instances. */ - /* Ensure the traceback is set correctly on the existing exception */ - if (tb != NULL) { - PyException_SetTraceback(val, tb); - Py_DECREF(tb); - } - va_start(vargs, format); msg_prefix = PyUnicode_FromFormatV(format, vargs); va_end(vargs); if (msg_prefix == NULL) { Py_DECREF(exc); - Py_DECREF(val); return NULL; } - PyErr_Format(exc, "%U (%s: %S)", - msg_prefix, Py_TYPE(val)->tp_name, val); - Py_DECREF(exc); + PyErr_Format((PyObject*)Py_TYPE(exc), "%U (%s: %S)", + msg_prefix, Py_TYPE(exc)->tp_name, exc); Py_DECREF(msg_prefix); - PyErr_Fetch(&new_exc, &new_val, &new_tb); - PyErr_NormalizeException(&new_exc, &new_val, &new_tb); - PyException_SetCause(new_val, val); - PyErr_Restore(new_exc, new_val, new_tb); - return new_val; + PyObject *new_exc = PyErr_GetRaisedException(); + PyException_SetCause(new_exc, exc); + PyErr_SetRaisedException(new_exc); + return new_exc; } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 34143c9a40b293e..133c991bf701c4e 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1308,7 +1308,6 @@ _PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear) /* Merge locals into fast locals */ PyObject *locals; PyObject **fast; - PyObject *error_type, *error_value, *error_traceback; PyCodeObject *co; locals = frame->f_locals; if (locals == NULL) { @@ -1317,7 +1316,7 @@ _PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear) fast = _PyFrame_GetLocalsArray(frame); co = frame->f_code; - PyErr_Fetch(&error_type, &error_value, &error_traceback); + PyObject *exc = PyErr_GetRaisedException(); for (int i = 0; i < co->co_nlocalsplus; i++) { _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); @@ -1374,7 +1373,7 @@ _PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear) } Py_XDECREF(value); } - PyErr_Restore(error_type, error_value, error_traceback); + PyErr_SetRaisedException(exc); } void diff --git a/Objects/genobject.c b/Objects/genobject.c index be08a59ece6b7e7..61463774310f884 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -69,8 +69,6 @@ void _PyGen_Finalize(PyObject *self) { PyGenObject *gen = (PyGenObject *)self; - PyObject *res = NULL; - PyObject *error_type, *error_value, *error_traceback; if (gen->gi_frame_state >= FRAME_COMPLETED) { /* Generator isn't paused, so no need to close */ @@ -82,23 +80,22 @@ _PyGen_Finalize(PyObject *self) PyObject *finalizer = agen->ag_origin_or_finalizer; if (finalizer && !agen->ag_closed) { /* Save the current exception, if any. */ - PyErr_Fetch(&error_type, &error_value, &error_traceback); - - res = PyObject_CallOneArg(finalizer, self); + PyObject *exc = PyErr_GetRaisedException(); + PyObject *res = PyObject_CallOneArg(finalizer, self); if (res == NULL) { PyErr_WriteUnraisable(self); } else { Py_DECREF(res); } /* Restore the saved exception. */ - PyErr_Restore(error_type, error_value, error_traceback); + PyErr_SetRaisedException(exc); return; } } /* Save the current exception, if any. */ - PyErr_Fetch(&error_type, &error_value, &error_traceback); + PyObject *exc = PyErr_GetRaisedException(); /* If `gen` is a coroutine, and if it was never awaited on, issue a RuntimeWarning. */ @@ -109,20 +106,19 @@ _PyGen_Finalize(PyObject *self) _PyErr_WarnUnawaitedCoroutine((PyObject *)gen); } else { - res = gen_close(gen, NULL); - } - - if (res == NULL) { - if (PyErr_Occurred()) { - PyErr_WriteUnraisable(self); + PyObject *res = gen_close(gen, NULL); + if (res == NULL) { + if (PyErr_Occurred()) { + PyErr_WriteUnraisable(self); + } + } + else { + Py_DECREF(res); } - } - else { - Py_DECREF(res); } /* Restore the saved exception. */ - PyErr_Restore(error_type, error_value, error_traceback); + PyErr_SetRaisedException(exc); } static void @@ -648,39 +644,11 @@ _PyGen_SetStopIterationValue(PyObject *value) int _PyGen_FetchStopIterationValue(PyObject **pvalue) { - PyObject *et, *ev, *tb; PyObject *value = NULL; - if (PyErr_ExceptionMatches(PyExc_StopIteration)) { - PyErr_Fetch(&et, &ev, &tb); - if (ev) { - /* exception will usually be normalised already */ - if (PyObject_TypeCheck(ev, (PyTypeObject *) et)) { - value = Py_NewRef(((PyStopIterationObject *)ev)->value); - Py_DECREF(ev); - } else if (et == PyExc_StopIteration && !PyTuple_Check(ev)) { - /* Avoid normalisation and take ev as value. - * - * Normalization is required if the value is a tuple, in - * that case the value of StopIteration would be set to - * the first element of the tuple. - * - * (See _PyErr_CreateException code for details.) - */ - value = ev; - } else { - /* normalisation required */ - PyErr_NormalizeException(&et, &ev, &tb); - if (!PyObject_TypeCheck(ev, (PyTypeObject *)PyExc_StopIteration)) { - PyErr_Restore(et, ev, tb); - return -1; - } - value = Py_NewRef(((PyStopIterationObject *)ev)->value); - Py_DECREF(ev); - } - } - Py_XDECREF(et); - Py_XDECREF(tb); + PyObject *exc = PyErr_GetRaisedException(); + value = Py_NewRef(((PyStopIterationObject *)exc)->value); + Py_DECREF(exc); } else if (PyErr_Occurred()) { return -1; } diff --git a/Objects/object.c b/Objects/object.c index 446c7b1f5f0302e..5db2b6af21ef135 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -370,13 +370,12 @@ _PyObject_Dump(PyObject* op) fflush(stderr); PyGILState_STATE gil = PyGILState_Ensure(); - PyObject *error_type, *error_value, *error_traceback; - PyErr_Fetch(&error_type, &error_value, &error_traceback); + PyObject *exc = PyErr_GetRaisedException(); (void)PyObject_Print(op, stderr, 0); fflush(stderr); - PyErr_Restore(error_type, error_value, error_traceback); + PyErr_SetRaisedException(exc); PyGILState_Release(gil); fprintf(stderr, "\n"); @@ -860,25 +859,22 @@ set_attribute_error_context(PyObject* v, PyObject* name) return 0; } // Intercept AttributeError exceptions and augment them to offer suggestions later. - PyObject *type, *value, *traceback; - PyErr_Fetch(&type, &value, &traceback); - PyErr_NormalizeException(&type, &value, &traceback); - // Check if the normalized exception is indeed an AttributeError - if (!PyErr_GivenExceptionMatches(value, PyExc_AttributeError)) { + PyObject *exc = PyErr_GetRaisedException(); + if (!PyErr_GivenExceptionMatches(exc, PyExc_AttributeError)) { goto restore; } - PyAttributeErrorObject* the_exc = (PyAttributeErrorObject*) value; + PyAttributeErrorObject* the_exc = (PyAttributeErrorObject*) exc; // Check if this exception was already augmented if (the_exc->name || the_exc->obj) { goto restore; } // Augment the exception with the name and object - if (PyObject_SetAttr(value, &_Py_ID(name), name) || - PyObject_SetAttr(value, &_Py_ID(obj), v)) { + if (PyObject_SetAttr(exc, &_Py_ID(name), name) || + PyObject_SetAttr(exc, &_Py_ID(obj), v)) { return 1; } restore: - PyErr_Restore(type, value, traceback); + PyErr_SetRaisedException(exc); return 0; } @@ -2190,9 +2186,8 @@ Py_ReprLeave(PyObject *obj) PyObject *dict; PyObject *list; Py_ssize_t i; - PyObject *error_type, *error_value, *error_traceback; - PyErr_Fetch(&error_type, &error_value, &error_traceback); + PyObject *exc = PyErr_GetRaisedException(); dict = PyThreadState_GetDict(); if (dict == NULL) @@ -2213,7 +2208,7 @@ Py_ReprLeave(PyObject *obj) finally: /* ignore exceptions because there is no way to report them. */ - PyErr_Restore(error_type, error_value, error_traceback); + PyErr_SetRaisedException(exc); } /* Trashcan support. */ diff --git a/Objects/odictobject.c b/Objects/odictobject.c index 215a8af54fb266b..39b0f684510578b 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -1556,10 +1556,9 @@ _PyODict_SetItem_KnownHash(PyObject *od, PyObject *key, PyObject *value, res = _odict_add_new_node((PyODictObject *)od, key, hash); if (res < 0) { /* Revert setting the value on the dict */ - PyObject *exc, *val, *tb; - PyErr_Fetch(&exc, &val, &tb); + PyObject *exc = PyErr_GetRaisedException(); (void) _PyDict_DelItem_KnownHash(od, key, hash); - _PyErr_ChainExceptions(exc, val, tb); + _PyErr_ChainExceptions1(exc); } } return res; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 981930f58417d8a..f486b83fd69e648 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4397,10 +4397,9 @@ static void type_dealloc_common(PyTypeObject *type) { if (type->tp_bases != NULL) { - PyObject *tp, *val, *tb; - PyErr_Fetch(&tp, &val, &tb); + PyObject *exc = PyErr_GetRaisedException(); remove_all_subclasses(type, type->tp_bases); - PyErr_Restore(tp, val, tb); + PyErr_SetRaisedException(exc); } } @@ -8445,10 +8444,9 @@ slot_tp_finalize(PyObject *self) { int unbound; PyObject *del, *res; - PyObject *error_type, *error_value, *error_traceback; /* Save the current exception, if any. */ - PyErr_Fetch(&error_type, &error_value, &error_traceback); + PyObject *exc = PyErr_GetRaisedException(); /* Execute __del__ method, if any. */ del = lookup_maybe_method(self, &_Py_ID(__del__), &unbound); @@ -8462,7 +8460,7 @@ slot_tp_finalize(PyObject *self) } /* Restore the saved exception. */ - PyErr_Restore(error_type, error_value, error_traceback); + PyErr_SetRaisedException(exc); } static PyObject * diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index bd7720e27533078..5a3e49a6fe45e33 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -959,9 +959,8 @@ PyObject_ClearWeakRefs(PyObject *object) if (*list != NULL) { PyWeakReference *current = *list; Py_ssize_t count = _PyWeakref_GetWeakrefCount(current); - PyObject *err_type, *err_value, *err_tb; + PyObject *exc = PyErr_GetRaisedException(); - PyErr_Fetch(&err_type, &err_value, &err_tb); if (count == 1) { PyObject *callback = current->wr_callback; @@ -980,7 +979,7 @@ PyObject_ClearWeakRefs(PyObject *object) tuple = PyTuple_New(count * 2); if (tuple == NULL) { - _PyErr_ChainExceptions(err_type, err_value, err_tb); + _PyErr_ChainExceptions1(exc); return; } @@ -1010,7 +1009,7 @@ PyObject_ClearWeakRefs(PyObject *object) Py_DECREF(tuple); } assert(!PyErr_Occurred()); - PyErr_Restore(err_type, err_value, err_tb); + PyErr_SetRaisedException(exc); } } From cbb0aa71d040022db61390380b8aebc7c04f3275 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 8 Mar 2023 12:03:50 -0700 Subject: [PATCH 42/97] gh-102304: Consolidate Direct Usage of _Py_RefTotal (gh-102514) This simplifies further changes to _Py_RefTotal (e.g. make it atomic or move it to PyInterpreterState). https://github.com/python/cpython/issues/102304 --- Include/cpython/object.h | 1 + Include/internal/pycore_object.h | 23 ++++++++-- Include/object.h | 25 ++++++++-- Modules/_testcapimodule.c | 7 +-- Objects/bytesobject.c | 9 ++-- Objects/dictobject.c | 10 ++-- Objects/object.c | 79 +++++++++++++++++++++++++++----- Objects/structseq.c | 2 +- Objects/tupleobject.c | 9 ++-- Objects/unicodeobject.c | 7 +-- 10 files changed, 127 insertions(+), 45 deletions(-) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 3f26f2487d70cc8..7b687d311359c38 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -3,6 +3,7 @@ #endif PyAPI_FUNC(void) _Py_NewReference(PyObject *op); +PyAPI_FUNC(void) _Py_NewReferenceNoTotal(PyObject *op); #ifdef Py_TRACE_REFS /* Py_TRACE_REFS is such major surgery that we call external routines. */ diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 8796dfe2f6b8cf6..e15685f174ebcfe 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -37,11 +37,23 @@ PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc( #define _Py_FatalRefcountError(message) \ _Py_FatalRefcountErrorFunc(__func__, (message)) + +#ifdef Py_REF_DEBUG +/* The symbol is only exposed in the API for the sake of extensions + built against the pre-3.12 stable ABI. */ +PyAPI_DATA(Py_ssize_t) _Py_RefTotal; + +extern void _Py_AddRefTotal(Py_ssize_t); +extern void _Py_IncRefTotal(void); +extern void _Py_DecRefTotal(void); +# define _Py_DEC_REFTOTAL() _Py_RefTotal-- +#endif + // Increment reference count by n static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n) { #ifdef Py_REF_DEBUG - _Py_RefTotal += n; + _Py_AddRefTotal(n); #endif op->ob_refcnt += n; } @@ -52,7 +64,7 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) { _Py_DECREF_STAT_INC(); #ifdef Py_REF_DEBUG - _Py_RefTotal--; + _Py_DEC_REFTOTAL(); #endif if (--op->ob_refcnt != 0) { assert(op->ob_refcnt > 0); @@ -70,7 +82,7 @@ _Py_DECREF_NO_DEALLOC(PyObject *op) { _Py_DECREF_STAT_INC(); #ifdef Py_REF_DEBUG - _Py_RefTotal--; + _Py_DEC_REFTOTAL(); #endif op->ob_refcnt--; #ifdef Py_DEBUG @@ -80,6 +92,11 @@ _Py_DECREF_NO_DEALLOC(PyObject *op) #endif } +#ifdef Py_REF_DEBUG +# undef _Py_DEC_REFTOTAL +#endif + + PyAPI_FUNC(int) _PyType_CheckConsistency(PyTypeObject *type); PyAPI_FUNC(int) _PyDict_CheckConsistency(PyObject *mp, int check_content); diff --git a/Include/object.h b/Include/object.h index 3774f1267300054..844b9c4a51c3e47 100644 --- a/Include/object.h +++ b/Include/object.h @@ -490,7 +490,21 @@ you can count such references to the type object.) */ #ifdef Py_REF_DEBUG -PyAPI_DATA(Py_ssize_t) _Py_RefTotal; +# if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 0x030A0000 +extern Py_ssize_t _Py_RefTotal; +# define _Py_INC_REFTOTAL() _Py_RefTotal++ +# define _Py_DEC_REFTOTAL() _Py_RefTotal-- +# elif defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +extern void _Py_IncRefTotal(void); +extern void _Py_DecRefTotal(void); +# define _Py_INC_REFTOTAL() _Py_IncRefTotal() +# define _Py_DEC_REFTOTAL() _Py_DecRefTotal() +# elif !defined(Py_LIMITED_API) || Py_LIMITED_API+0 > 0x030C0000 +extern void _Py_IncRefTotal_DO_NOT_USE_THIS(void); +extern void _Py_DecRefTotal_DO_NOT_USE_THIS(void); +# define _Py_INC_REFTOTAL() _Py_IncRefTotal_DO_NOT_USE_THIS() +# define _Py_DEC_REFTOTAL() _Py_DecRefTotal_DO_NOT_USE_THIS() +# endif PyAPI_FUNC(void) _Py_NegativeRefcount(const char *filename, int lineno, PyObject *op); #endif /* Py_REF_DEBUG */ @@ -519,8 +533,8 @@ static inline void Py_INCREF(PyObject *op) // Non-limited C API and limited C API for Python 3.9 and older access // directly PyObject.ob_refcnt. #ifdef Py_REF_DEBUG - _Py_RefTotal++; -#endif + _Py_INC_REFTOTAL(); +#endif // Py_REF_DEBUG op->ob_refcnt++; #endif } @@ -539,7 +553,7 @@ static inline void Py_DECREF(PyObject *op) { static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) { _Py_DECREF_STAT_INC(); - _Py_RefTotal--; + _Py_DEC_REFTOTAL(); if (--op->ob_refcnt != 0) { if (op->ob_refcnt < 0) { _Py_NegativeRefcount(filename, lineno, op); @@ -564,6 +578,9 @@ static inline void Py_DECREF(PyObject *op) #define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) #endif +#undef _Py_INC_REFTOTAL +#undef _Py_DEC_REFTOTAL + /* Safely decref `op` and set `op` to NULL, especially useful in tp_clear * and tp_dealloc implementations. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 10e507d6b481de0..ea67017a1ba3b16 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1654,15 +1654,10 @@ slot_tp_del(PyObject *self) */ { Py_ssize_t refcnt = Py_REFCNT(self); - _Py_NewReference(self); + _Py_NewReferenceNoTotal(self); Py_SET_REFCNT(self, refcnt); } assert(!PyType_IS_GC(Py_TYPE(self)) || PyObject_GC_IsTracked(self)); - /* If Py_REF_DEBUG macro is defined, _Py_NewReference() increased - _Py_RefTotal, so we need to undo that. */ -#ifdef Py_REF_DEBUG - _Py_RefTotal--; -#endif } static PyObject * diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 657443f31fa709b..687a654bdae137c 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3060,21 +3060,20 @@ _PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) Py_DECREF(v); return 0; } - /* XXX UNREF/NEWREF interface should be more symmetrical */ -#ifdef Py_REF_DEBUG - _Py_RefTotal--; -#endif #ifdef Py_TRACE_REFS _Py_ForgetReference(v); #endif *pv = (PyObject *) PyObject_Realloc(v, PyBytesObject_SIZE + newsize); if (*pv == NULL) { +#ifdef Py_REF_DEBUG + _Py_DecRefTotal(); +#endif PyObject_Free(v); PyErr_NoMemory(); return -1; } - _Py_NewReference(*pv); + _Py_NewReferenceNoTotal(*pv); sv = (PyBytesObject *) *pv; Py_SET_SIZE(sv, newsize); sv->ob_sval[newsize] = '\0'; diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 75c92172a91778c..a60f275742a5341 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -303,7 +303,7 @@ static inline void dictkeys_incref(PyDictKeysObject *dk) { #ifdef Py_REF_DEBUG - _Py_RefTotal++; + _Py_IncRefTotal(); #endif dk->dk_refcnt++; } @@ -313,7 +313,7 @@ dictkeys_decref(PyDictKeysObject *dk) { assert(dk->dk_refcnt > 0); #ifdef Py_REF_DEBUG - _Py_RefTotal--; + _Py_DecRefTotal(); #endif if (--dk->dk_refcnt == 0) { free_keys_object(dk); @@ -633,7 +633,7 @@ new_keys_object(uint8_t log2_size, bool unicode) } } #ifdef Py_REF_DEBUG - _Py_RefTotal++; + _Py_IncRefTotal(); #endif dk->dk_refcnt = 1; dk->dk_log2_size = log2_size; @@ -821,7 +821,7 @@ clone_combined_dict_keys(PyDictObject *orig) we have it now; calling dictkeys_incref would be an error as keys->dk_refcnt is already set to 1 (after memcpy). */ #ifdef Py_REF_DEBUG - _Py_RefTotal++; + _Py_IncRefTotal(); #endif return keys; } @@ -1520,7 +1520,7 @@ dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode) // We can not use free_keys_object here because key's reference // are moved already. #ifdef Py_REF_DEBUG - _Py_RefTotal--; + _Py_DecRefTotal(); #endif if (oldkeys == Py_EMPTY_KEYS) { oldkeys->dk_refcnt--; diff --git a/Objects/object.c b/Objects/object.c index 5db2b6af21ef135..38da4d497a96e72 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -56,6 +56,24 @@ _PyObject_CheckConsistency(PyObject *op, int check_content) #ifdef Py_REF_DEBUG Py_ssize_t _Py_RefTotal; +static inline void +reftotal_increment(void) +{ + _Py_RefTotal++; +} + +static inline void +reftotal_decrement(void) +{ + _Py_RefTotal--; +} + +void +_Py_AddRefTotal(Py_ssize_t n) +{ + _Py_RefTotal += n; +} + Py_ssize_t _Py_GetRefTotal(void) { @@ -121,6 +139,32 @@ _Py_NegativeRefcount(const char *filename, int lineno, PyObject *op) filename, lineno, __func__); } +/* This is exposed strictly for use in Py_INCREF(). */ +PyAPI_FUNC(void) +_Py_IncRefTotal_DO_NOT_USE_THIS(void) +{ + reftotal_increment(); +} + +/* This is exposed strictly for use in Py_DECREF(). */ +PyAPI_FUNC(void) +_Py_DecRefTotal_DO_NOT_USE_THIS(void) +{ + reftotal_decrement(); +} + +void +_Py_IncRefTotal(void) +{ + reftotal_increment(); +} + +void +_Py_DecRefTotal(void) +{ + reftotal_decrement(); +} + #endif /* Py_REF_DEBUG */ void @@ -138,12 +182,18 @@ Py_DecRef(PyObject *o) void _Py_IncRef(PyObject *o) { +#ifdef Py_REF_DEBUG + reftotal_increment(); +#endif Py_INCREF(o); } void _Py_DecRef(PyObject *o) { +#ifdef Py_REF_DEBUG + reftotal_decrement(); +#endif Py_DECREF(o); } @@ -238,17 +288,12 @@ PyObject_CallFinalizerFromDealloc(PyObject *self) /* tp_finalize resurrected it! Make it look like the original Py_DECREF * never happened. */ Py_ssize_t refcnt = Py_REFCNT(self); - _Py_NewReference(self); + _Py_NewReferenceNoTotal(self); Py_SET_REFCNT(self, refcnt); _PyObject_ASSERT(self, (!_PyType_IS_GC(Py_TYPE(self)) || _PyObject_GC_IS_TRACKED(self))); - /* If Py_REF_DEBUG macro is defined, _Py_NewReference() increased - _Py_RefTotal, so we need to undo that. */ -#ifdef Py_REF_DEBUG - _Py_RefTotal--; -#endif return -1; } @@ -2010,21 +2055,33 @@ _PyTypes_FiniTypes(PyInterpreterState *interp) } -void -_Py_NewReference(PyObject *op) +static inline void +new_reference(PyObject *op) { if (_PyRuntime.tracemalloc.config.tracing) { _PyTraceMalloc_NewReference(op); } -#ifdef Py_REF_DEBUG - _Py_RefTotal++; -#endif Py_SET_REFCNT(op, 1); #ifdef Py_TRACE_REFS _Py_AddToAllObjects(op, 1); #endif } +void +_Py_NewReference(PyObject *op) +{ +#ifdef Py_REF_DEBUG + reftotal_increment(); +#endif + new_reference(op); +} + +void +_Py_NewReferenceNoTotal(PyObject *op) +{ + new_reference(op); +} + #ifdef Py_TRACE_REFS void diff --git a/Objects/structseq.c b/Objects/structseq.c index 100ccfef0a23c41..c20962ecd82563c 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -592,7 +592,7 @@ _PyStructSequence_FiniType(PyTypeObject *type) // Don't use Py_DECREF(): static type must not be deallocated Py_SET_REFCNT(type, 0); #ifdef Py_REF_DEBUG - _Py_RefTotal--; + _Py_DecRefTotal(); #endif // Make sure that _PyStructSequence_InitType() will initialize diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 6ee93ab5adc2816..59c0251639d3dda 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -930,10 +930,6 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) return *pv == NULL ? -1 : 0; } - /* XXX UNREF/NEWREF interface should be more symmetrical */ -#ifdef Py_REF_DEBUG - _Py_RefTotal--; -#endif if (_PyObject_GC_IS_TRACKED(v)) { _PyObject_GC_UNTRACK(v); } @@ -947,10 +943,13 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) sv = PyObject_GC_Resize(PyTupleObject, v, newsize); if (sv == NULL) { *pv = NULL; +#ifdef Py_REF_DEBUG + _Py_DecRefTotal(); +#endif PyObject_GC_Del(v); return -1; } - _Py_NewReference((PyObject *) sv); + _Py_NewReferenceNoTotal((PyObject *) sv); /* Zero out items added by growing */ if (newsize > oldsize) memset(&sv->ob_item[oldsize], 0, diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 1ba30421c66dba4..2d50f9c340f2f36 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -947,21 +947,18 @@ resize_compact(PyObject *unicode, Py_ssize_t length) _PyUnicode_UTF8(unicode) = NULL; _PyUnicode_UTF8_LENGTH(unicode) = 0; } -#ifdef Py_REF_DEBUG - _Py_RefTotal--; -#endif #ifdef Py_TRACE_REFS _Py_ForgetReference(unicode); #endif new_unicode = (PyObject *)PyObject_Realloc(unicode, new_size); if (new_unicode == NULL) { - _Py_NewReference(unicode); + _Py_NewReferenceNoTotal(unicode); PyErr_NoMemory(); return NULL; } unicode = new_unicode; - _Py_NewReference(unicode); + _Py_NewReferenceNoTotal(unicode); _PyUnicode_LENGTH(unicode) = length; #ifdef Py_DEBUG From 66ff374d4f353ae427c148d2a1d141d223303a82 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 8 Mar 2023 15:56:36 -0700 Subject: [PATCH 43/97] gh-100227: Move func_state.next_version to PyInterpreterState (gh-102334) https://github.com/python/cpython/issues/100227 --- Include/internal/pycore_function.h | 2 +- Include/internal/pycore_interp.h | 1 + Include/internal/pycore_runtime.h | 2 -- Include/internal/pycore_runtime_init.h | 6 +++--- Objects/funcobject.c | 5 +++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h index 5cedb33d7e3afdb..11988149843fefc 100644 --- a/Include/internal/pycore_function.h +++ b/Include/internal/pycore_function.h @@ -10,7 +10,7 @@ extern "C" { #define FUNC_MAX_WATCHERS 8 -struct _py_func_runtime_state { +struct _py_func_state { uint32_t next_version; }; diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 7ef9c40153e4211..9efed0a1cf90c27 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -141,6 +141,7 @@ struct _is { struct _Py_float_state float_state; struct _Py_long_state long_state; struct _dtoa_state dtoa; + struct _py_func_state func_state; /* Using a cache is very effective since typically only a single slice is created and then deleted again. */ PySliceObject *slice_cache; diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 2350eaab5976caf..e0e3d4ace0cfde1 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -13,7 +13,6 @@ extern "C" { #include "pycore_dict_state.h" // struct _Py_dict_runtime_state #include "pycore_floatobject.h" // struct _Py_float_runtime_state #include "pycore_faulthandler.h" // struct _faulthandler_runtime_state -#include "pycore_function.h" // struct _func_runtime_state #include "pycore_global_objects.h" // struct _Py_global_objects #include "pycore_import.h" // struct _import_runtime_state #include "pycore_interp.h" // PyInterpreterState @@ -155,7 +154,6 @@ typedef struct pyruntimestate { struct _Py_float_runtime_state float_state; struct _Py_unicode_runtime_state unicode_state; struct _Py_dict_runtime_state dict_state; - struct _py_func_runtime_state func_state; struct { /* Used to set PyTypeObject.tp_version_tag */ diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index a2cc7c87c2f3e29..aeabcfd2b9056ca 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -68,9 +68,6 @@ extern PyTypeObject _PyExc_MemoryError; .dict_state = { \ .next_keys_version = 2, \ }, \ - .func_state = { \ - .next_version = 1, \ - }, \ .types = { \ .next_version_tag = 1, \ }, \ @@ -116,6 +113,9 @@ extern PyTypeObject _PyExc_MemoryError; }, \ }, \ .dtoa = _dtoa_state_INIT(&(INTERP)), \ + .func_state = { \ + .next_version = 1, \ + }, \ .static_objects = { \ .singletons = { \ ._not_used = 1, \ diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 99048ea41c6c800..ce5d7bda32c0320 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -227,10 +227,11 @@ uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func) if (func->vectorcall != _PyFunction_Vectorcall) { return 0; } - if (_PyRuntime.func_state.next_version == 0) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (interp->func_state.next_version == 0) { return 0; } - uint32_t v = _PyRuntime.func_state.next_version++; + uint32_t v = interp->func_state.next_version++; func->func_version = v; return v; } From 5e5acd291f4387876afc641163e9f8ae5c65086c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 8 Mar 2023 18:04:16 -0700 Subject: [PATCH 44/97] gh-100227: Move next_keys_version to PyInterpreterState (gh-102335) https://github.com/python/cpython/issues/100227 --- Include/internal/pycore_dict.h | 3 ++- Include/internal/pycore_dict_state.h | 4 +++- Include/internal/pycore_runtime_init.h | 6 +++--- Objects/dictobject.c | 7 ++++--- Python/specialize.c | 16 +++++++++++----- 5 files changed, 23 insertions(+), 13 deletions(-) diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 1af5e59a677a9a1..12c3c708e29b955 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -37,7 +37,8 @@ extern PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *); /* Gets a version number unique to the current state of the keys of dict, if possible. * Returns the version number, or zero if it was not possible to get a version number. */ -extern uint32_t _PyDictKeys_GetVersionForCurrentState(PyDictKeysObject *dictkeys); +extern uint32_t _PyDictKeys_GetVersionForCurrentState( + PyInterpreterState *interp, PyDictKeysObject *dictkeys); extern size_t _PyDict_KeysSize(PyDictKeysObject *keys); diff --git a/Include/internal/pycore_dict_state.h b/Include/internal/pycore_dict_state.h index 77375ea8beb8777..c5142ee6763a86f 100644 --- a/Include/internal/pycore_dict_state.h +++ b/Include/internal/pycore_dict_state.h @@ -14,7 +14,6 @@ struct _Py_dict_runtime_state { * It is incremented each time that a dictionary is created and each * time that a dictionary is modified. */ uint64_t global_version; - uint32_t next_keys_version; }; @@ -30,6 +29,8 @@ struct _Py_dict_runtime_state { #define DICT_MAX_WATCHERS 8 struct _Py_dict_state { + uint32_t next_keys_version; + #if PyDict_MAXFREELIST > 0 /* Dictionary reuse scheme to save calls to malloc and free */ PyDictObject *free_list[PyDict_MAXFREELIST]; @@ -37,6 +38,7 @@ struct _Py_dict_state { int numfree; int keys_numfree; #endif + PyDict_WatchCallback watchers[DICT_MAX_WATCHERS]; }; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index aeabcfd2b9056ca..efc82b43a61dab1 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -65,9 +65,6 @@ extern PyTypeObject _PyExc_MemoryError; .float_format = _py_float_format_unknown, \ .double_format = _py_float_format_unknown, \ }, \ - .dict_state = { \ - .next_keys_version = 2, \ - }, \ .types = { \ .next_version_tag = 1, \ }, \ @@ -113,6 +110,9 @@ extern PyTypeObject _PyExc_MemoryError; }, \ }, \ .dtoa = _dtoa_state_INIT(&(INTERP)), \ + .dict_state = { \ + .next_keys_version = 2, \ + }, \ .func_state = { \ .next_version = 1, \ }, \ diff --git a/Objects/dictobject.c b/Objects/dictobject.c index a60f275742a5341..b58e93f4320d425 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5655,15 +5655,16 @@ _PyDictKeys_DecRef(PyDictKeysObject *keys) dictkeys_decref(keys); } -uint32_t _PyDictKeys_GetVersionForCurrentState(PyDictKeysObject *dictkeys) +uint32_t _PyDictKeys_GetVersionForCurrentState(PyInterpreterState *interp, + PyDictKeysObject *dictkeys) { if (dictkeys->dk_version != 0) { return dictkeys->dk_version; } - if (_PyRuntime.dict_state.next_keys_version == 0) { + if (interp->dict_state.next_keys_version == 0) { return 0; } - uint32_t v = _PyRuntime.dict_state.next_keys_version++; + uint32_t v = interp->dict_state.next_keys_version++; dictkeys->dk_version = v; return v; } diff --git a/Python/specialize.c b/Python/specialize.c index 3405d2b0ab06803..0a7af8991eec83c 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -512,7 +512,8 @@ specialize_module_load_attr( SPEC_FAIL_OUT_OF_RANGE); return -1; } - uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(dict->ma_keys); + uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState( + _PyInterpreterState_GET(), dict->ma_keys); if (keys_version == 0) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); return -1; @@ -1063,7 +1064,8 @@ PyObject *descr, DescriptorClassification kind) SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_SHADOWED); return 0; } - uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(keys); + uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState( + _PyInterpreterState_GET(), keys); if (keys_version == 0) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); return 0; @@ -1134,12 +1136,14 @@ _Py_Specialize_LoadGlobal( SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_EXPECTED_ERROR); goto fail; } + PyInterpreterState *interp = _PyInterpreterState_GET(); if (index != DKIX_EMPTY) { if (index != (uint16_t)index) { SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE); goto fail; } - uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(globals_keys); + uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState( + interp, globals_keys); if (keys_version == 0) { SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS); goto fail; @@ -1167,12 +1171,14 @@ _Py_Specialize_LoadGlobal( SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE); goto fail; } - uint32_t globals_version = _PyDictKeys_GetVersionForCurrentState(globals_keys); + uint32_t globals_version = _PyDictKeys_GetVersionForCurrentState( + interp, globals_keys); if (globals_version == 0) { SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS); goto fail; } - uint32_t builtins_version = _PyDictKeys_GetVersionForCurrentState(builtin_keys); + uint32_t builtins_version = _PyDictKeys_GetVersionForCurrentState( + interp, builtin_keys); if (builtins_version == 0) { SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS); goto fail; From 58d761e5b5f253892a697941c71bac0159a1d74e Mon Sep 17 00:00:00 2001 From: "T. Wouters" Date: Wed, 8 Mar 2023 18:39:33 -0800 Subject: [PATCH 45/97] GH-84783: Document GH-101264 (Make the slice object hashable) in What's New. (#102548) --- Doc/whatsnew/3.12.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index c0c021c679147f8..d982cb62ec2f4e3 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -189,6 +189,8 @@ Other Language Changes part of comprehensions (like ``a``) is still disallowed, as per :pep:`572`. (Contributed by Nikita Sobolev in :gh:`100581`.) +* :class:`slice` objects are now hashable, allowing them to be used as dict keys and + set items. (Contributed by Furkan Onder in :gh:`101264`.) New Modules =========== From b45d14b88611fefc6f054226d3e1117082d322c8 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 9 Mar 2023 08:16:30 -0700 Subject: [PATCH 46/97] gh-100227: Move dict_state.global_version to PyInterpreterState (gh-102338) https://github.com/python/cpython/issues/100227 --- Include/internal/pycore_dict.h | 11 +- Include/internal/pycore_dict_state.h | 12 +- Include/internal/pycore_runtime.h | 2 - Objects/dictobject.c | 236 +++++++++++++++++---------- Python/bytecodes.c | 4 +- Python/generated_cases.c.h | 4 +- 6 files changed, 160 insertions(+), 109 deletions(-) diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 12c3c708e29b955..6253e0841ad3492 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -149,8 +149,8 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) { #define DICT_VERSION_INCREMENT (1 << DICT_MAX_WATCHERS) #define DICT_VERSION_MASK (DICT_VERSION_INCREMENT - 1) -#define DICT_NEXT_VERSION() \ - (_PyRuntime.dict_state.global_version += DICT_VERSION_INCREMENT) +#define DICT_NEXT_VERSION(INTERP) \ + ((INTERP)->dict_state.global_version += DICT_VERSION_INCREMENT) void _PyDict_SendEvent(int watcher_bits, @@ -160,7 +160,8 @@ _PyDict_SendEvent(int watcher_bits, PyObject *value); static inline uint64_t -_PyDict_NotifyEvent(PyDict_WatchEvent event, +_PyDict_NotifyEvent(PyInterpreterState *interp, + PyDict_WatchEvent event, PyDictObject *mp, PyObject *key, PyObject *value) @@ -169,9 +170,9 @@ _PyDict_NotifyEvent(PyDict_WatchEvent event, int watcher_bits = mp->ma_version_tag & DICT_VERSION_MASK; if (watcher_bits) { _PyDict_SendEvent(watcher_bits, event, mp, key, value); - return DICT_NEXT_VERSION() | watcher_bits; + return DICT_NEXT_VERSION(interp) | watcher_bits; } - return DICT_NEXT_VERSION(); + return DICT_NEXT_VERSION(interp); } extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values); diff --git a/Include/internal/pycore_dict_state.h b/Include/internal/pycore_dict_state.h index c5142ee6763a86f..d608088ed2d7cec 100644 --- a/Include/internal/pycore_dict_state.h +++ b/Include/internal/pycore_dict_state.h @@ -9,14 +9,6 @@ extern "C" { #endif -struct _Py_dict_runtime_state { - /*Global counter used to set ma_version_tag field of dictionary. - * It is incremented each time that a dictionary is created and each - * time that a dictionary is modified. */ - uint64_t global_version; -}; - - #ifndef WITH_FREELISTS // without freelists # define PyDict_MAXFREELIST 0 @@ -29,6 +21,10 @@ struct _Py_dict_runtime_state { #define DICT_MAX_WATCHERS 8 struct _Py_dict_state { + /*Global counter used to set ma_version_tag field of dictionary. + * It is incremented each time that a dictionary is created and each + * time that a dictionary is modified. */ + uint64_t global_version; uint32_t next_keys_version; #if PyDict_MAXFREELIST > 0 diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index e0e3d4ace0cfde1..520109ca440444b 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -10,7 +10,6 @@ extern "C" { #include "pycore_atomic.h" /* _Py_atomic_address */ #include "pycore_ceval_state.h" // struct _ceval_runtime_state -#include "pycore_dict_state.h" // struct _Py_dict_runtime_state #include "pycore_floatobject.h" // struct _Py_float_runtime_state #include "pycore_faulthandler.h" // struct _faulthandler_runtime_state #include "pycore_global_objects.h" // struct _Py_global_objects @@ -153,7 +152,6 @@ typedef struct pyruntimestate { struct _Py_float_runtime_state float_state; struct _Py_unicode_runtime_state unicode_state; - struct _Py_dict_runtime_state dict_state; struct { /* Used to set PyTypeObject.tp_version_tag */ diff --git a/Objects/dictobject.c b/Objects/dictobject.c index b58e93f4320d425..227e438a8dfffce 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -232,7 +232,8 @@ equally good collision statistics, needed less code & used less memory. */ -static int dictresize(PyDictObject *mp, uint8_t log_newsize, int unicode); +static int dictresize(PyInterpreterState *interp, PyDictObject *mp, + uint8_t log_newsize, int unicode); static PyObject* dict_iter(PyDictObject *dict); @@ -241,9 +242,8 @@ static PyObject* dict_iter(PyDictObject *dict); #if PyDict_MAXFREELIST > 0 static struct _Py_dict_state * -get_dict_state(void) +get_dict_state(PyInterpreterState *interp) { - PyInterpreterState *interp = _PyInterpreterState_GET(); return &interp->dict_state; } #endif @@ -289,7 +289,8 @@ void _PyDict_DebugMallocStats(FILE *out) { #if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(); + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _Py_dict_state *state = get_dict_state(interp); _PyDebugAllocatorStats(out, "free PyDictObject", state->numfree, sizeof(PyDictObject)); #endif @@ -297,7 +298,7 @@ _PyDict_DebugMallocStats(FILE *out) #define DK_MASK(dk) (DK_SIZE(dk)-1) -static void free_keys_object(PyDictKeysObject *keys); +static void free_keys_object(PyInterpreterState *interp, PyDictKeysObject *keys); static inline void dictkeys_incref(PyDictKeysObject *dk) @@ -309,14 +310,14 @@ dictkeys_incref(PyDictKeysObject *dk) } static inline void -dictkeys_decref(PyDictKeysObject *dk) +dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk) { assert(dk->dk_refcnt > 0); #ifdef Py_REF_DEBUG _Py_DecRefTotal(); #endif if (--dk->dk_refcnt == 0) { - free_keys_object(dk); + free_keys_object(interp, dk); } } @@ -586,7 +587,7 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) static PyDictKeysObject* -new_keys_object(uint8_t log2_size, bool unicode) +new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) { PyDictKeysObject *dk; Py_ssize_t usable; @@ -612,7 +613,7 @@ new_keys_object(uint8_t log2_size, bool unicode) } #if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(); + struct _Py_dict_state *state = get_dict_state(interp); #ifdef Py_DEBUG // new_keys_object() must not be called after _PyDict_Fini() assert(state->keys_numfree != -1); @@ -648,7 +649,7 @@ new_keys_object(uint8_t log2_size, bool unicode) } static void -free_keys_object(PyDictKeysObject *keys) +free_keys_object(PyInterpreterState *interp, PyDictKeysObject *keys) { assert(keys != Py_EMPTY_KEYS); if (DK_IS_UNICODE(keys)) { @@ -668,7 +669,7 @@ free_keys_object(PyDictKeysObject *keys) } } #if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(); + struct _Py_dict_state *state = get_dict_state(interp); #ifdef Py_DEBUG // free_keys_object() must not be called after _PyDict_Fini() assert(state->keys_numfree != -1); @@ -709,12 +710,14 @@ free_values(PyDictValues *values) /* Consumes a reference to the keys object */ static PyObject * -new_dict(PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, int free_values_on_failure) +new_dict(PyInterpreterState *interp, + PyDictKeysObject *keys, PyDictValues *values, + Py_ssize_t used, int free_values_on_failure) { PyDictObject *mp; assert(keys != NULL); #if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(); + struct _Py_dict_state *state = get_dict_state(interp); #ifdef Py_DEBUG // new_dict() must not be called after _PyDict_Fini() assert(state->numfree != -1); @@ -731,7 +734,7 @@ new_dict(PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, int free { mp = PyObject_GC_New(PyDictObject, &PyDict_Type); if (mp == NULL) { - dictkeys_decref(keys); + dictkeys_decref(interp, keys); if (free_values_on_failure) { free_values(values); } @@ -741,7 +744,7 @@ new_dict(PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, int free mp->ma_keys = keys; mp->ma_values = values; mp->ma_used = used; - mp->ma_version_tag = DICT_NEXT_VERSION(); + mp->ma_version_tag = DICT_NEXT_VERSION(interp); ASSERT_CONSISTENT(mp); return (PyObject *)mp; } @@ -754,19 +757,19 @@ shared_keys_usable_size(PyDictKeysObject *keys) /* Consumes a reference to the keys object */ static PyObject * -new_dict_with_shared_keys(PyDictKeysObject *keys) +new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys) { size_t size = shared_keys_usable_size(keys); PyDictValues *values = new_values(size); if (values == NULL) { - dictkeys_decref(keys); + dictkeys_decref(interp, keys); return PyErr_NoMemory(); } ((char *)values)[-2] = 0; for (size_t i = 0; i < size; i++) { values->values[i] = NULL; } - return new_dict(keys, values, 0, 1); + return new_dict(interp, keys, values, 0, 1); } @@ -829,8 +832,9 @@ clone_combined_dict_keys(PyDictObject *orig) PyObject * PyDict_New(void) { + PyInterpreterState *interp = _PyInterpreterState_GET(); dictkeys_incref(Py_EMPTY_KEYS); - return new_dict(Py_EMPTY_KEYS, NULL, 0, 0); + return new_dict(interp, Py_EMPTY_KEYS, NULL, 0, 0); } /* Search index of hash table from offset of entry table */ @@ -1170,9 +1174,9 @@ find_empty_slot(PyDictKeysObject *keys, Py_hash_t hash) } static int -insertion_resize(PyDictObject *mp, int unicode) +insertion_resize(PyInterpreterState *interp, PyDictObject *mp, int unicode) { - return dictresize(mp, calculate_log2_keysize(GROWTH_RATE(mp)), unicode); + return dictresize(interp, mp, calculate_log2_keysize(GROWTH_RATE(mp)), unicode); } static Py_ssize_t @@ -1214,12 +1218,13 @@ Returns -1 if an error occurred, or 0 on success. Consumes key and value references. */ static int -insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) +insertdict(PyInterpreterState *interp, PyDictObject *mp, + PyObject *key, Py_hash_t hash, PyObject *value) { PyObject *old_value; if (DK_IS_UNICODE(mp->ma_keys) && !PyUnicode_CheckExact(key)) { - if (insertion_resize(mp, 0) < 0) + if (insertion_resize(interp, mp, 0) < 0) goto Fail; assert(mp->ma_keys->dk_kind == DICT_KEYS_GENERAL); } @@ -1231,13 +1236,14 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) MAINTAIN_TRACKING(mp, key, value); if (ix == DKIX_EMPTY) { - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_ADDED, mp, key, value); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_ADDED, mp, key, value); /* Insert into new slot. */ mp->ma_keys->dk_version = 0; assert(old_value == NULL); if (mp->ma_keys->dk_usable <= 0) { /* Need to resize. */ - if (insertion_resize(mp, 1) < 0) + if (insertion_resize(interp, mp, 1) < 0) goto Fail; } @@ -1275,7 +1281,8 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) } if (old_value != value) { - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, mp, key, value); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_MODIFIED, mp, key, value); if (_PyDict_HasSplitTable(mp)) { mp->ma_values->values[ix] = value; if (old_value == NULL) { @@ -1308,21 +1315,23 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) // Same to insertdict but specialized for ma_keys = Py_EMPTY_KEYS. // Consumes key and value references. static int -insert_to_emptydict(PyDictObject *mp, PyObject *key, Py_hash_t hash, - PyObject *value) +insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, + PyObject *key, Py_hash_t hash, PyObject *value) { assert(mp->ma_keys == Py_EMPTY_KEYS); - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_ADDED, mp, key, value); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_ADDED, mp, key, value); int unicode = PyUnicode_CheckExact(key); - PyDictKeysObject *newkeys = new_keys_object(PyDict_LOG_MINSIZE, unicode); + PyDictKeysObject *newkeys = new_keys_object( + interp, PyDict_LOG_MINSIZE, unicode); if (newkeys == NULL) { Py_DECREF(key); Py_DECREF(value); return -1; } - dictkeys_decref(Py_EMPTY_KEYS); + dictkeys_decref(interp, Py_EMPTY_KEYS); mp->ma_keys = newkeys; mp->ma_values = NULL; @@ -1397,7 +1406,8 @@ This function supports: - Generic -> Generic */ static int -dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode) +dictresize(PyInterpreterState *interp, PyDictObject *mp, + uint8_t log2_newsize, int unicode) { PyDictKeysObject *oldkeys; PyDictValues *oldvalues; @@ -1421,7 +1431,7 @@ dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode) */ /* Allocate a new table. */ - mp->ma_keys = new_keys_object(log2_newsize, unicode); + mp->ma_keys = new_keys_object(interp, log2_newsize, unicode); if (mp->ma_keys == NULL) { mp->ma_keys = oldkeys; return -1; @@ -1462,7 +1472,7 @@ dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode) } build_indices_unicode(mp->ma_keys, newentries, numentries); } - dictkeys_decref(oldkeys); + dictkeys_decref(interp, oldkeys); mp->ma_values = NULL; free_values(oldvalues); } @@ -1530,7 +1540,7 @@ dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode) assert(oldkeys->dk_kind != DICT_KEYS_SPLIT); assert(oldkeys->dk_refcnt == 1); #if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(); + struct _Py_dict_state *state = get_dict_state(interp); #ifdef Py_DEBUG // dictresize() must not be called after _PyDict_Fini() assert(state->keys_numfree != -1); @@ -1557,7 +1567,7 @@ dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode) } static PyObject * -dict_new_presized(Py_ssize_t minused, bool unicode) +dict_new_presized(PyInterpreterState *interp, Py_ssize_t minused, bool unicode) { const uint8_t log2_max_presize = 17; const Py_ssize_t max_presize = ((Py_ssize_t)1) << log2_max_presize; @@ -1578,16 +1588,17 @@ dict_new_presized(Py_ssize_t minused, bool unicode) log2_newsize = estimate_log2_keysize(minused); } - new_keys = new_keys_object(log2_newsize, unicode); + new_keys = new_keys_object(interp, log2_newsize, unicode); if (new_keys == NULL) return NULL; - return new_dict(new_keys, NULL, 0, 0); + return new_dict(interp, new_keys, NULL, 0, 0); } PyObject * _PyDict_NewPresized(Py_ssize_t minused) { - return dict_new_presized(minused, false); + PyInterpreterState *interp = _PyInterpreterState_GET(); + return dict_new_presized(interp, minused, false); } PyObject * @@ -1597,6 +1608,7 @@ _PyDict_FromItems(PyObject *const *keys, Py_ssize_t keys_offset, { bool unicode = true; PyObject *const *ks = keys; + PyInterpreterState *interp = _PyInterpreterState_GET(); for (Py_ssize_t i = 0; i < length; i++) { if (!PyUnicode_CheckExact(*ks)) { @@ -1606,7 +1618,7 @@ _PyDict_FromItems(PyObject *const *keys, Py_ssize_t keys_offset, ks += keys_offset; } - PyObject *dict = dict_new_presized(length, unicode); + PyObject *dict = dict_new_presized(interp, length, unicode); if (dict == NULL) { return NULL; } @@ -1834,11 +1846,12 @@ _PyDict_SetItem_Take2(PyDictObject *mp, PyObject *key, PyObject *value) return -1; } } + PyInterpreterState *interp = _PyInterpreterState_GET(); if (mp->ma_keys == Py_EMPTY_KEYS) { - return insert_to_emptydict(mp, key, hash, value); + return insert_to_emptydict(interp, mp, key, hash, value); } /* insertdict() handles any resizing that might be necessary */ - return insertdict(mp, key, hash, value); + return insertdict(interp, mp, key, hash, value); } /* CAUTION: PyDict_SetItem() must guarantee that it won't resize the @@ -1875,11 +1888,12 @@ _PyDict_SetItem_KnownHash(PyObject *op, PyObject *key, PyObject *value, assert(hash != -1); mp = (PyDictObject *)op; + PyInterpreterState *interp = _PyInterpreterState_GET(); if (mp->ma_keys == Py_EMPTY_KEYS) { - return insert_to_emptydict(mp, Py_NewRef(key), hash, Py_NewRef(value)); + return insert_to_emptydict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); } /* insertdict() handles any resizing that might be necessary */ - return insertdict(mp, Py_NewRef(key), hash, Py_NewRef(value)); + return insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); } static void @@ -1977,7 +1991,9 @@ _PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) return -1; } - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_DELETED, mp, key, NULL); + PyInterpreterState *interp = _PyInterpreterState_GET(); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_DELETED, mp, key, NULL); return delitem_common(mp, hash, ix, old_value, new_version); } @@ -2020,7 +2036,9 @@ _PyDict_DelItemIf(PyObject *op, PyObject *key, assert(hashpos >= 0); if (res > 0) { - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_DELETED, mp, key, NULL); + PyInterpreterState *interp = _PyInterpreterState_GET(); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_DELETED, mp, key, NULL); return delitem_common(mp, hashpos, ix, old_value, new_version); } else { return 0; @@ -2045,7 +2063,9 @@ PyDict_Clear(PyObject *op) return; } /* Empty the dict... */ - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_CLEARED, mp, NULL, NULL); + PyInterpreterState *interp = _PyInterpreterState_GET(); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_CLEARED, mp, NULL, NULL); dictkeys_incref(Py_EMPTY_KEYS); mp->ma_keys = Py_EMPTY_KEYS; mp->ma_values = NULL; @@ -2057,11 +2077,11 @@ PyDict_Clear(PyObject *op) for (i = 0; i < n; i++) Py_CLEAR(oldvalues->values[i]); free_values(oldvalues); - dictkeys_decref(oldkeys); + dictkeys_decref(interp, oldkeys); } else { assert(oldkeys->dk_refcnt == 1); - dictkeys_decref(oldkeys); + dictkeys_decref(interp, oldkeys); } ASSERT_CONSISTENT(mp); } @@ -2165,6 +2185,7 @@ _PyDict_Pop_KnownHash(PyObject *dict, PyObject *key, Py_hash_t hash, PyObject *d Py_ssize_t ix; PyObject *old_value; PyDictObject *mp; + PyInterpreterState *interp = _PyInterpreterState_GET(); assert(PyDict_Check(dict)); mp = (PyDictObject *)dict; @@ -2187,7 +2208,8 @@ _PyDict_Pop_KnownHash(PyObject *dict, PyObject *key, Py_hash_t hash, PyObject *d return NULL; } assert(old_value != NULL); - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_DELETED, mp, key, NULL); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_DELETED, mp, key, NULL); delitem_common(mp, hash, ix, Py_NewRef(old_value), new_version); ASSERT_CONSISTENT(mp); @@ -2222,6 +2244,7 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) PyObject *key; PyObject *d; int status; + PyInterpreterState *interp = _PyInterpreterState_GET(); d = _PyObject_CallNoArgs(cls); if (d == NULL) @@ -2236,13 +2259,16 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) Py_hash_t hash; int unicode = DK_IS_UNICODE(((PyDictObject*)iterable)->ma_keys); - if (dictresize(mp, estimate_log2_keysize(PyDict_GET_SIZE(iterable)), unicode)) { + if (dictresize(interp, mp, + estimate_log2_keysize(PyDict_GET_SIZE(iterable)), + unicode)) { Py_DECREF(d); return NULL; } while (_PyDict_Next(iterable, &pos, &key, &oldvalue, &hash)) { - if (insertdict(mp, Py_NewRef(key), hash, Py_NewRef(value))) { + if (insertdict(interp, mp, + Py_NewRef(key), hash, Py_NewRef(value))) { Py_DECREF(d); return NULL; } @@ -2255,13 +2281,14 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) PyObject *key; Py_hash_t hash; - if (dictresize(mp, estimate_log2_keysize(PySet_GET_SIZE(iterable)), 0)) { + if (dictresize(interp, mp, + estimate_log2_keysize(PySet_GET_SIZE(iterable)), 0)) { Py_DECREF(d); return NULL; } while (_PySet_NextEntry(iterable, &pos, &key, &hash)) { - if (insertdict(mp, Py_NewRef(key), hash, Py_NewRef(value))) { + if (insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value))) { Py_DECREF(d); return NULL; } @@ -2308,9 +2335,10 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) static void dict_dealloc(PyDictObject *mp) { + PyInterpreterState *interp = _PyInterpreterState_GET(); assert(Py_REFCNT(mp) == 0); Py_SET_REFCNT(mp, 1); - _PyDict_NotifyEvent(PyDict_EVENT_DEALLOCATED, mp, NULL, NULL); + _PyDict_NotifyEvent(interp, PyDict_EVENT_DEALLOCATED, mp, NULL, NULL); if (Py_REFCNT(mp) > 1) { Py_SET_REFCNT(mp, Py_REFCNT(mp) - 1); return; @@ -2328,14 +2356,14 @@ dict_dealloc(PyDictObject *mp) Py_XDECREF(values->values[i]); } free_values(values); - dictkeys_decref(keys); + dictkeys_decref(interp, keys); } else if (keys != NULL) { assert(keys->dk_refcnt == 1 || keys == Py_EMPTY_KEYS); - dictkeys_decref(keys); + dictkeys_decref(interp, keys); } #if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(); + struct _Py_dict_state *state = get_dict_state(interp); #ifdef Py_DEBUG // new_dict() must not be called after _PyDict_Fini() assert(state->numfree != -1); @@ -2765,7 +2793,7 @@ PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) } static int -dict_merge(PyObject *a, PyObject *b, int override) +dict_merge(PyInterpreterState *interp, PyObject *a, PyObject *b, int override) { PyDictObject *mp, *other; @@ -2799,13 +2827,14 @@ dict_merge(PyObject *a, PyObject *b, int override) other->ma_used == okeys->dk_nentries && (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE || USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)) { - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_CLONED, mp, b, NULL); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_CLONED, mp, b, NULL); PyDictKeysObject *keys = clone_combined_dict_keys(other); if (keys == NULL) { return -1; } - dictkeys_decref(mp->ma_keys); + dictkeys_decref(interp, mp->ma_keys); mp->ma_keys = keys; if (mp->ma_values != NULL) { free_values(mp->ma_values); @@ -2830,7 +2859,9 @@ dict_merge(PyObject *a, PyObject *b, int override) */ if (USABLE_FRACTION(DK_SIZE(mp->ma_keys)) < other->ma_used) { int unicode = DK_IS_UNICODE(other->ma_keys); - if (dictresize(mp, estimate_log2_keysize(mp->ma_used + other->ma_used), unicode)) { + if (dictresize(interp, mp, + estimate_log2_keysize(mp->ma_used + other->ma_used), + unicode)) { return -1; } } @@ -2845,12 +2876,14 @@ dict_merge(PyObject *a, PyObject *b, int override) Py_INCREF(key); Py_INCREF(value); if (override == 1) { - err = insertdict(mp, Py_NewRef(key), hash, Py_NewRef(value)); + err = insertdict(interp, mp, + Py_NewRef(key), hash, Py_NewRef(value)); } else { err = _PyDict_Contains_KnownHash(a, key, hash); if (err == 0) { - err = insertdict(mp, Py_NewRef(key), hash, Py_NewRef(value)); + err = insertdict(interp, mp, + Py_NewRef(key), hash, Py_NewRef(value)); } else if (err > 0) { if (override != 0) { @@ -2936,20 +2969,23 @@ dict_merge(PyObject *a, PyObject *b, int override) int PyDict_Update(PyObject *a, PyObject *b) { - return dict_merge(a, b, 1); + PyInterpreterState *interp = _PyInterpreterState_GET(); + return dict_merge(interp, a, b, 1); } int PyDict_Merge(PyObject *a, PyObject *b, int override) { + PyInterpreterState *interp = _PyInterpreterState_GET(); /* XXX Deprecate override not in (0, 1). */ - return dict_merge(a, b, override != 0); + return dict_merge(interp, a, b, override != 0); } int _PyDict_MergeEx(PyObject *a, PyObject *b, int override) { - return dict_merge(a, b, override); + PyInterpreterState *interp = _PyInterpreterState_GET(); + return dict_merge(interp, a, b, override); } static PyObject * @@ -2963,6 +2999,7 @@ PyDict_Copy(PyObject *o) { PyObject *copy; PyDictObject *mp; + PyInterpreterState *interp = _PyInterpreterState_GET(); if (o == NULL || !PyDict_Check(o)) { PyErr_BadInternalCall(); @@ -2991,7 +3028,7 @@ PyDict_Copy(PyObject *o) split_copy->ma_values = newvalues; split_copy->ma_keys = mp->ma_keys; split_copy->ma_used = mp->ma_used; - split_copy->ma_version_tag = DICT_NEXT_VERSION(); + split_copy->ma_version_tag = DICT_NEXT_VERSION(interp); dictkeys_incref(mp->ma_keys); for (size_t i = 0; i < size; i++) { PyObject *value = mp->ma_values->values[i]; @@ -3024,7 +3061,7 @@ PyDict_Copy(PyObject *o) if (keys == NULL) { return NULL; } - PyDictObject *new = (PyDictObject *)new_dict(keys, NULL, 0, 0); + PyDictObject *new = (PyDictObject *)new_dict(interp, keys, NULL, 0, 0); if (new == NULL) { /* In case of an error, `new_dict()` takes care of cleaning up `keys`. */ @@ -3044,7 +3081,7 @@ PyDict_Copy(PyObject *o) copy = PyDict_New(); if (copy == NULL) return NULL; - if (dict_merge(copy, o, 1) == 0) + if (dict_merge(interp, copy, o, 1) == 0) return copy; Py_DECREF(copy); return NULL; @@ -3244,6 +3281,7 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) PyDictObject *mp = (PyDictObject *)d; PyObject *value; Py_hash_t hash; + PyInterpreterState *interp = _PyInterpreterState_GET(); if (!PyDict_Check(d)) { PyErr_BadInternalCall(); @@ -3257,7 +3295,7 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) } if (mp->ma_keys == Py_EMPTY_KEYS) { - if (insert_to_emptydict(mp, Py_NewRef(key), hash, + if (insert_to_emptydict(interp, mp, Py_NewRef(key), hash, Py_NewRef(defaultobj)) < 0) { return NULL; } @@ -3265,7 +3303,7 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) } if (!PyUnicode_CheckExact(key) && DK_IS_UNICODE(mp->ma_keys)) { - if (insertion_resize(mp, 0) < 0) { + if (insertion_resize(interp, mp, 0) < 0) { return NULL; } } @@ -3275,11 +3313,12 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) return NULL; if (ix == DKIX_EMPTY) { - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_ADDED, mp, key, defaultobj); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_ADDED, mp, key, defaultobj); mp->ma_keys->dk_version = 0; value = defaultobj; if (mp->ma_keys->dk_usable <= 0) { - if (insertion_resize(mp, 1) < 0) { + if (insertion_resize(interp, mp, 1) < 0) { return NULL; } } @@ -3314,7 +3353,8 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) assert(mp->ma_keys->dk_usable >= 0); } else if (value == NULL) { - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_ADDED, mp, key, defaultobj); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_ADDED, mp, key, defaultobj); value = defaultobj; assert(_PyDict_HasSplitTable(mp)); assert(mp->ma_values->values[ix] == NULL); @@ -3395,6 +3435,7 @@ dict_popitem_impl(PyDictObject *self) Py_ssize_t i, j; PyObject *res; uint64_t new_version; + PyInterpreterState *interp = _PyInterpreterState_GET(); /* Allocate the result tuple before checking the size. Believe it * or not, this allocation could trigger a garbage collection which @@ -3415,7 +3456,7 @@ dict_popitem_impl(PyDictObject *self) } /* Convert split table to combined table */ if (self->ma_keys->dk_kind == DICT_KEYS_SPLIT) { - if (dictresize(self, DK_LOG_SIZE(self->ma_keys), 1)) { + if (dictresize(interp, self, DK_LOG_SIZE(self->ma_keys), 1)) { Py_DECREF(res); return NULL; } @@ -3434,7 +3475,8 @@ dict_popitem_impl(PyDictObject *self) assert(i >= 0); key = ep0[i].me_key; - new_version = _PyDict_NotifyEvent(PyDict_EVENT_DELETED, self, key, NULL); + new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_DELETED, self, key, NULL); hash = unicode_get_hash(key); value = ep0[i].me_value; ep0[i].me_key = NULL; @@ -3449,7 +3491,8 @@ dict_popitem_impl(PyDictObject *self) assert(i >= 0); key = ep0[i].me_key; - new_version = _PyDict_NotifyEvent(PyDict_EVENT_DELETED, self, key, NULL); + new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_DELETED, self, key, NULL); hash = ep0[i].me_hash; value = ep0[i].me_value; ep0[i].me_key = NULL; @@ -3708,7 +3751,8 @@ dict_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyDictObject *d = (PyDictObject *)self; d->ma_used = 0; - d->ma_version_tag = DICT_NEXT_VERSION(); + d->ma_version_tag = DICT_NEXT_VERSION( + _PyInterpreterState_GET()); dictkeys_incref(Py_EMPTY_KEYS); d->ma_keys = Py_EMPTY_KEYS; d->ma_values = NULL; @@ -5262,7 +5306,9 @@ dictvalues_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)) PyDictKeysObject * _PyDict_NewKeysForClass(void) { - PyDictKeysObject *keys = new_keys_object(NEXT_LOG2_SHARED_KEYS_MAX_SIZE, 1); + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyDictKeysObject *keys = new_keys_object( + interp, NEXT_LOG2_SHARED_KEYS_MAX_SIZE, 1); if (keys == NULL) { PyErr_Clear(); } @@ -5306,6 +5352,7 @@ init_inline_values(PyObject *obj, PyTypeObject *tp) int _PyObject_InitializeDict(PyObject *obj) { + PyInterpreterState *interp = _PyInterpreterState_GET(); PyTypeObject *tp = Py_TYPE(obj); if (tp->tp_dictoffset == 0) { return 0; @@ -5317,7 +5364,7 @@ _PyObject_InitializeDict(PyObject *obj) PyObject *dict; if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { dictkeys_incref(CACHED_KEYS(tp)); - dict = new_dict_with_shared_keys(CACHED_KEYS(tp)); + dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); } else { dict = PyDict_New(); @@ -5331,7 +5378,8 @@ _PyObject_InitializeDict(PyObject *obj) } static PyObject * -make_dict_from_instance_attributes(PyDictKeysObject *keys, PyDictValues *values) +make_dict_from_instance_attributes(PyInterpreterState *interp, + PyDictKeysObject *keys, PyDictValues *values) { dictkeys_incref(keys); Py_ssize_t used = 0; @@ -5344,7 +5392,7 @@ make_dict_from_instance_attributes(PyDictKeysObject *keys, PyDictValues *values) track += _PyObject_GC_MAY_BE_TRACKED(val); } } - PyObject *res = new_dict(keys, values, used, 0); + PyObject *res = new_dict(interp, keys, values, used, 0); if (track && res) { _PyObject_GC_TRACK(res); } @@ -5354,15 +5402,17 @@ make_dict_from_instance_attributes(PyDictKeysObject *keys, PyDictValues *values) PyObject * _PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values) { + PyInterpreterState *interp = _PyInterpreterState_GET(); PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); OBJECT_STAT_INC(dict_materialized_on_request); - return make_dict_from_instance_attributes(keys, values); + return make_dict_from_instance_attributes(interp, keys, values); } int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, PyObject *name, PyObject *value) { + PyInterpreterState *interp = _PyInterpreterState_GET(); PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); assert(keys != NULL); assert(values != NULL); @@ -5385,7 +5435,8 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, OBJECT_STAT_INC(dict_materialized_str_subclass); } #endif - PyObject *dict = make_dict_from_instance_attributes(keys, values); + PyObject *dict = make_dict_from_instance_attributes( + interp, keys, values); if (dict == NULL) { return -1; } @@ -5564,13 +5615,15 @@ PyObject * PyObject_GenericGetDict(PyObject *obj, void *context) { PyObject *dict; + PyInterpreterState *interp = _PyInterpreterState_GET(); PyTypeObject *tp = Py_TYPE(obj); if (_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)) { PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); if (_PyDictOrValues_IsValues(*dorv_ptr)) { PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); OBJECT_STAT_INC(dict_materialized_on_request); - dict = make_dict_from_instance_attributes(CACHED_KEYS(tp), values); + dict = make_dict_from_instance_attributes( + interp, CACHED_KEYS(tp), values); if (dict != NULL) { dorv_ptr->dict = dict; } @@ -5579,7 +5632,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context) dict = _PyDictOrValues_GetDict(*dorv_ptr); if (dict == NULL) { dictkeys_incref(CACHED_KEYS(tp)); - dict = new_dict_with_shared_keys(CACHED_KEYS(tp)); + dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); dorv_ptr->dict = dict; } } @@ -5596,7 +5649,8 @@ PyObject_GenericGetDict(PyObject *obj, void *context) PyTypeObject *tp = Py_TYPE(obj); if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { dictkeys_incref(CACHED_KEYS(tp)); - *dictptr = dict = new_dict_with_shared_keys(CACHED_KEYS(tp)); + *dictptr = dict = new_dict_with_shared_keys( + interp, CACHED_KEYS(tp)); } else { *dictptr = dict = PyDict_New(); @@ -5613,6 +5667,7 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, PyObject *dict; int res; PyDictKeysObject *cached; + PyInterpreterState *interp = _PyInterpreterState_GET(); assert(dictptr != NULL); if ((tp->tp_flags & Py_TPFLAGS_HEAPTYPE) && (cached = CACHED_KEYS(tp))) { @@ -5620,7 +5675,7 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, dict = *dictptr; if (dict == NULL) { dictkeys_incref(cached); - dict = new_dict_with_shared_keys(cached); + dict = new_dict_with_shared_keys(interp, cached); if (dict == NULL) return -1; *dictptr = dict; @@ -5652,7 +5707,8 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, void _PyDictKeys_DecRef(PyDictKeysObject *keys) { - dictkeys_decref(keys); + PyInterpreterState *interp = _PyInterpreterState_GET(); + dictkeys_decref(interp, keys); } uint32_t _PyDictKeys_GetVersionForCurrentState(PyInterpreterState *interp, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 63dbecad3b45fc8..45d507266743211 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1679,7 +1679,7 @@ dummy_func( DEOPT_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } else { @@ -1687,7 +1687,7 @@ dummy_func( DEOPT_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } Py_DECREF(old_value); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 82e18505b0d4306..51357cda4c8949c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2114,7 +2114,7 @@ DEOPT_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } else { @@ -2122,7 +2122,7 @@ DEOPT_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } Py_DECREF(old_value); From cf6e7c5e551b3513817d6a77ba88253dc8473298 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 9 Mar 2023 09:46:21 -0700 Subject: [PATCH 47/97] gh-100227: Isolate the Import State to Each Interpreter (gh-101941) Specific changes: * move the import lock to PyInterpreterState * move the "find_and_load" diagnostic state to PyInterpreterState Note that the import lock exists to keep multiple imports of the same module in the same interpreter (but in different threads) from stomping on each other. Independently, we use a distinct global lock to protect globally shared import state, especially related to loaded extension modules. For now we can rely on the GIL as that lock but with a per-interpreter GIL we'll need a new global lock. The remaining state in _PyRuntimeState.imports will (probably) continue being global. https://github.com/python/cpython/issues/100227 --- Include/cpython/import.h | 4 +- Include/internal/pycore_import.h | 34 +++++---- Include/internal/pycore_runtime_init.h | 10 --- Modules/posixmodule.c | 13 ++-- Python/import.c | 98 ++++++++++++++------------ 5 files changed, 85 insertions(+), 74 deletions(-) diff --git a/Include/cpython/import.h b/Include/cpython/import.h index a58801b47f1bec2..2bca4ade4c4f2cf 100644 --- a/Include/cpython/import.h +++ b/Include/cpython/import.h @@ -10,8 +10,8 @@ PyAPI_FUNC(PyObject *) _PyImport_GetModuleId(_Py_Identifier *name); PyAPI_FUNC(int) _PyImport_SetModule(PyObject *name, PyObject *module); PyAPI_FUNC(int) _PyImport_SetModuleString(const char *name, PyObject* module); -PyAPI_FUNC(void) _PyImport_AcquireLock(void); -PyAPI_FUNC(int) _PyImport_ReleaseLock(void); +PyAPI_FUNC(void) _PyImport_AcquireLock(PyInterpreterState *interp); +PyAPI_FUNC(int) _PyImport_ReleaseLock(PyInterpreterState *interp); PyAPI_FUNC(int) _PyImport_FixupBuiltin( PyObject *mod, diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index b7ffe01c0c0e207..69ed6273b7e6091 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -21,17 +21,6 @@ struct _import_runtime_state { This is initialized lazily in _PyImport_FixupExtensionObject(). Modules are added there and looked up in _imp.find_extension(). */ PyObject *extensions; - /* The global import lock. */ - struct { - PyThread_type_lock mutex; - unsigned long thread; - int level; - } lock; - struct { - int import_level; - _PyTime_t accumulated; - int header; - } find_and_load; /* Package context -- the full module name for package imports */ const char * pkgcontext; }; @@ -69,6 +58,18 @@ struct _import_state { int dlopenflags; #endif PyObject *import_func; + /* The global import lock. */ + struct { + PyThread_type_lock mutex; + unsigned long thread; + int level; + } lock; + /* diagnostic info in PyImport_ImportModuleLevelObject() */ + struct { + int import_level; + _PyTime_t accumulated; + int header; + } find_and_load; }; #ifdef HAVE_DLOPEN @@ -86,8 +87,15 @@ struct _import_state { #define IMPORTS_INIT \ { \ - .override_frozen_modules = 0, \ DLOPENFLAGS_INIT \ + .lock = { \ + .mutex = NULL, \ + .thread = PYTHREAD_INVALID_THREAD_ID, \ + .level = 0, \ + }, \ + .find_and_load = { \ + .header = 1, \ + }, \ } extern void _PyImport_ClearCore(PyInterpreterState *interp); @@ -138,7 +146,7 @@ extern void _PyImport_FiniExternal(PyInterpreterState *interp); #ifdef HAVE_FORK -extern PyStatus _PyImport_ReInitLock(void); +extern PyStatus _PyImport_ReInitLock(PyInterpreterState *interp); #endif diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index efc82b43a61dab1..bdecac944dfd3a2 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -40,16 +40,6 @@ extern PyTypeObject _PyExc_MemoryError; in accordance with the specification. */ \ .autoTSSkey = Py_tss_NEEDS_INIT, \ .parser = _parser_runtime_state_INIT, \ - .imports = { \ - .lock = { \ - .mutex = NULL, \ - .thread = PYTHREAD_INVALID_THREAD_ID, \ - .level = 0, \ - }, \ - .find_and_load = { \ - .header = 1, \ - }, \ - }, \ .ceval = { \ .perf = _PyEval_RUNTIME_PERF_INIT, \ }, \ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 7beb2cee64a05c6..a3d86cbe7a57fb2 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -567,18 +567,21 @@ run_at_forkers(PyObject *lst, int reverse) void PyOS_BeforeFork(void) { - run_at_forkers(_PyInterpreterState_GET()->before_forkers, 1); + PyInterpreterState *interp = _PyInterpreterState_GET(); + run_at_forkers(interp->before_forkers, 1); - _PyImport_AcquireLock(); + _PyImport_AcquireLock(interp); } void PyOS_AfterFork_Parent(void) { - if (_PyImport_ReleaseLock() <= 0) + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_PyImport_ReleaseLock(interp) <= 0) { Py_FatalError("failed releasing import lock after fork"); + } - run_at_forkers(_PyInterpreterState_GET()->after_forkers_parent, 0); + run_at_forkers(interp->after_forkers_parent, 0); } void @@ -604,7 +607,7 @@ PyOS_AfterFork_Child(void) goto fatal_error; } - status = _PyImport_ReInitLock(); + status = _PyImport_ReInitLock(tstate->interp); if (_PyStatus_EXCEPTION(status)) { goto fatal_error; } diff --git a/Python/import.c b/Python/import.c index 57d4eea148810f8..1bf4199e125aa5d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -56,11 +56,6 @@ static struct _inittab *inittab_copy = NULL; #define LAST_MODULE_INDEX _PyRuntime.imports.last_module_index #define EXTENSIONS _PyRuntime.imports.extensions -#define import_lock _PyRuntime.imports.lock.mutex -#define import_lock_thread _PyRuntime.imports.lock.thread -#define import_lock_level _PyRuntime.imports.lock.level - -#define FIND_AND_LOAD _PyRuntime.imports.find_and_load #define PKGCONTEXT (_PyRuntime.imports.pkgcontext) @@ -85,6 +80,16 @@ static struct _inittab *inittab_copy = NULL; #define IMPORT_FUNC(interp) \ (interp)->imports.import_func +#define IMPORT_LOCK(interp) \ + (interp)->imports.lock.mutex +#define IMPORT_LOCK_THREAD(interp) \ + (interp)->imports.lock.thread +#define IMPORT_LOCK_LEVEL(interp) \ + (interp)->imports.lock.level + +#define FIND_AND_LOAD(interp) \ + (interp)->imports.find_and_load + /*******************/ /* the import lock */ @@ -95,45 +100,45 @@ static struct _inittab *inittab_copy = NULL; These calls are serialized by the global interpreter lock. */ void -_PyImport_AcquireLock(void) +_PyImport_AcquireLock(PyInterpreterState *interp) { unsigned long me = PyThread_get_thread_ident(); if (me == PYTHREAD_INVALID_THREAD_ID) return; /* Too bad */ - if (import_lock == NULL) { - import_lock = PyThread_allocate_lock(); - if (import_lock == NULL) + if (IMPORT_LOCK(interp) == NULL) { + IMPORT_LOCK(interp) = PyThread_allocate_lock(); + if (IMPORT_LOCK(interp) == NULL) return; /* Nothing much we can do. */ } - if (import_lock_thread == me) { - import_lock_level++; + if (IMPORT_LOCK_THREAD(interp) == me) { + IMPORT_LOCK_LEVEL(interp)++; return; } - if (import_lock_thread != PYTHREAD_INVALID_THREAD_ID || - !PyThread_acquire_lock(import_lock, 0)) + if (IMPORT_LOCK_THREAD(interp) != PYTHREAD_INVALID_THREAD_ID || + !PyThread_acquire_lock(IMPORT_LOCK(interp), 0)) { PyThreadState *tstate = PyEval_SaveThread(); - PyThread_acquire_lock(import_lock, WAIT_LOCK); + PyThread_acquire_lock(IMPORT_LOCK(interp), WAIT_LOCK); PyEval_RestoreThread(tstate); } - assert(import_lock_level == 0); - import_lock_thread = me; - import_lock_level = 1; + assert(IMPORT_LOCK_LEVEL(interp) == 0); + IMPORT_LOCK_THREAD(interp) = me; + IMPORT_LOCK_LEVEL(interp) = 1; } int -_PyImport_ReleaseLock(void) +_PyImport_ReleaseLock(PyInterpreterState *interp) { unsigned long me = PyThread_get_thread_ident(); - if (me == PYTHREAD_INVALID_THREAD_ID || import_lock == NULL) + if (me == PYTHREAD_INVALID_THREAD_ID || IMPORT_LOCK(interp) == NULL) return 0; /* Too bad */ - if (import_lock_thread != me) + if (IMPORT_LOCK_THREAD(interp) != me) return -1; - import_lock_level--; - assert(import_lock_level >= 0); - if (import_lock_level == 0) { - import_lock_thread = PYTHREAD_INVALID_THREAD_ID; - PyThread_release_lock(import_lock); + IMPORT_LOCK_LEVEL(interp)--; + assert(IMPORT_LOCK_LEVEL(interp) >= 0); + if (IMPORT_LOCK_LEVEL(interp) == 0) { + IMPORT_LOCK_THREAD(interp) = PYTHREAD_INVALID_THREAD_ID; + PyThread_release_lock(IMPORT_LOCK(interp)); } return 1; } @@ -144,23 +149,23 @@ _PyImport_ReleaseLock(void) We now acquire the import lock around fork() calls but on some platforms (Solaris 9 and earlier? see isue7242) that still left us with problems. */ PyStatus -_PyImport_ReInitLock(void) +_PyImport_ReInitLock(PyInterpreterState *interp) { - if (import_lock != NULL) { - if (_PyThread_at_fork_reinit(&import_lock) < 0) { + if (IMPORT_LOCK(interp) != NULL) { + if (_PyThread_at_fork_reinit(&IMPORT_LOCK(interp)) < 0) { return _PyStatus_ERR("failed to create a new lock"); } } - if (import_lock_level > 1) { + if (IMPORT_LOCK_LEVEL(interp) > 1) { /* Forked as a side effect of import */ unsigned long me = PyThread_get_thread_ident(); - PyThread_acquire_lock(import_lock, WAIT_LOCK); - import_lock_thread = me; - import_lock_level--; + PyThread_acquire_lock(IMPORT_LOCK(interp), WAIT_LOCK); + IMPORT_LOCK_THREAD(interp) = me; + IMPORT_LOCK_LEVEL(interp)--; } else { - import_lock_thread = PYTHREAD_INVALID_THREAD_ID; - import_lock_level = 0; + IMPORT_LOCK_THREAD(interp) = PYTHREAD_INVALID_THREAD_ID; + IMPORT_LOCK_LEVEL(interp) = 0; } return _PyStatus_OK(); } @@ -2506,8 +2511,8 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) PyObject *mod = NULL; PyInterpreterState *interp = tstate->interp; int import_time = _PyInterpreterState_GetConfig(interp)->import_time; -#define import_level FIND_AND_LOAD.import_level -#define accumulated FIND_AND_LOAD.accumulated +#define import_level FIND_AND_LOAD(interp).import_level +#define accumulated FIND_AND_LOAD(interp).accumulated _PyTime_t t1 = 0, accumulated_copy = accumulated; @@ -2528,7 +2533,7 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) * _PyDict_GetItemIdWithError(). */ if (import_time) { -#define header FIND_AND_LOAD.header +#define header FIND_AND_LOAD(interp).header if (header) { fputs("import time: self [us] | cumulative | imported package\n", stderr); @@ -2867,10 +2872,6 @@ _PyImport_Fini(void) { /* Destroy the database used by _PyImport_{Fixup,Find}Extension */ _extensions_cache_clear_all(); - if (import_lock != NULL) { - PyThread_free_lock(import_lock); - import_lock = NULL; - } /* Use the same memory allocator as _PyImport_Init(). */ PyMemAllocatorEx old_alloc; @@ -2959,6 +2960,11 @@ _PyImport_FiniCore(PyInterpreterState *interp) PyErr_WriteUnraisable(NULL); } + if (IMPORT_LOCK(interp) != NULL) { + PyThread_free_lock(IMPORT_LOCK(interp)); + IMPORT_LOCK(interp) = NULL; + } + _PyImport_ClearCore(interp); } @@ -3090,7 +3096,9 @@ static PyObject * _imp_lock_held_impl(PyObject *module) /*[clinic end generated code: output=8b89384b5e1963fc input=9b088f9b217d9bdf]*/ { - return PyBool_FromLong(import_lock_thread != PYTHREAD_INVALID_THREAD_ID); + PyInterpreterState *interp = _PyInterpreterState_GET(); + return PyBool_FromLong( + IMPORT_LOCK_THREAD(interp) != PYTHREAD_INVALID_THREAD_ID); } /*[clinic input] @@ -3106,7 +3114,8 @@ static PyObject * _imp_acquire_lock_impl(PyObject *module) /*[clinic end generated code: output=1aff58cb0ee1b026 input=4a2d4381866d5fdc]*/ { - _PyImport_AcquireLock(); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyImport_AcquireLock(interp); Py_RETURN_NONE; } @@ -3122,7 +3131,8 @@ static PyObject * _imp_release_lock_impl(PyObject *module) /*[clinic end generated code: output=7faab6d0be178b0a input=934fb11516dd778b]*/ { - if (_PyImport_ReleaseLock() < 0) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_PyImport_ReleaseLock(interp) < 0) { PyErr_SetString(PyExc_RuntimeError, "not holding the import lock"); return NULL; From ca066bdbed85094a9c4d9930823ce3587807db48 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 9 Mar 2023 09:50:33 -0700 Subject: [PATCH 48/97] gh-81057: Vendor a Subset of distutils for the c-analyzer Tool (gh-102505) distutils was removed in November. However, the c-analyzer relies on it. To solve that here, we vendor the parts the tool needs so it can be run against 3.12+. (Also see gh-92584.) Note that we may end up removing this code later in favor of a solution in common with the peg_generator tool (which also relies on distutils). At the least, the copy here makes sure the c-analyzer tool works on 3.12+ in the meantime. --- Tools/c-analyzer/distutils/README | 2 + Tools/c-analyzer/distutils/__init__.py | 0 Tools/c-analyzer/distutils/_msvccompiler.py | 203 ++++++++ Tools/c-analyzer/distutils/bcppcompiler.py | 109 ++++ Tools/c-analyzer/distutils/ccompiler.py | 470 ++++++++++++++++++ Tools/c-analyzer/distutils/cygwinccompiler.py | 286 +++++++++++ Tools/c-analyzer/distutils/debug.py | 5 + Tools/c-analyzer/distutils/dep_util.py | 29 ++ Tools/c-analyzer/distutils/errors.py | 48 ++ Tools/c-analyzer/distutils/log.py | 63 +++ Tools/c-analyzer/distutils/msvc9compiler.py | 438 ++++++++++++++++ Tools/c-analyzer/distutils/msvccompiler.py | 327 ++++++++++++ Tools/c-analyzer/distutils/spawn.py | 48 ++ Tools/c-analyzer/distutils/unixccompiler.py | 102 ++++ Tools/c-analyzer/distutils/util.py | 171 +++++++ 15 files changed, 2301 insertions(+) create mode 100644 Tools/c-analyzer/distutils/README create mode 100644 Tools/c-analyzer/distutils/__init__.py create mode 100644 Tools/c-analyzer/distutils/_msvccompiler.py create mode 100644 Tools/c-analyzer/distutils/bcppcompiler.py create mode 100644 Tools/c-analyzer/distutils/ccompiler.py create mode 100644 Tools/c-analyzer/distutils/cygwinccompiler.py create mode 100644 Tools/c-analyzer/distutils/debug.py create mode 100644 Tools/c-analyzer/distutils/dep_util.py create mode 100644 Tools/c-analyzer/distutils/errors.py create mode 100644 Tools/c-analyzer/distutils/log.py create mode 100644 Tools/c-analyzer/distutils/msvc9compiler.py create mode 100644 Tools/c-analyzer/distutils/msvccompiler.py create mode 100644 Tools/c-analyzer/distutils/spawn.py create mode 100644 Tools/c-analyzer/distutils/unixccompiler.py create mode 100644 Tools/c-analyzer/distutils/util.py diff --git a/Tools/c-analyzer/distutils/README b/Tools/c-analyzer/distutils/README new file mode 100644 index 000000000000000..b260b8e06fac06d --- /dev/null +++ b/Tools/c-analyzer/distutils/README @@ -0,0 +1,2 @@ +This is a partial copy of distutils as it was removed in 0faa0ba240e. +It only includes the parts needed by the C parser. diff --git a/Tools/c-analyzer/distutils/__init__.py b/Tools/c-analyzer/distutils/__init__.py new file mode 100644 index 000000000000000..e69de29bb2d1d64 diff --git a/Tools/c-analyzer/distutils/_msvccompiler.py b/Tools/c-analyzer/distutils/_msvccompiler.py new file mode 100644 index 000000000000000..1e67870d13323d3 --- /dev/null +++ b/Tools/c-analyzer/distutils/_msvccompiler.py @@ -0,0 +1,203 @@ +"""distutils._msvccompiler + +Contains MSVCCompiler, an implementation of the abstract CCompiler class +for Microsoft Visual Studio 2015. + +The module is compatible with VS 2015 and later. You can find legacy support +for older versions in distutils.msvc9compiler and distutils.msvccompiler. +""" + +# Written by Perry Stoll +# hacked by Robin Becker and Thomas Heller to do a better job of +# finding DevStudio (through the registry) +# ported to VS 2005 and VS 2008 by Christian Heimes +# ported to VS 2015 by Steve Dower + +import os +import subprocess +import winreg + +from distutils.errors import DistutilsPlatformError +from distutils.ccompiler import CCompiler +from distutils import log + +from itertools import count + +def _find_vc2015(): + try: + key = winreg.OpenKeyEx( + winreg.HKEY_LOCAL_MACHINE, + r"Software\Microsoft\VisualStudio\SxS\VC7", + access=winreg.KEY_READ | winreg.KEY_WOW64_32KEY + ) + except OSError: + log.debug("Visual C++ is not registered") + return None, None + + best_version = 0 + best_dir = None + with key: + for i in count(): + try: + v, vc_dir, vt = winreg.EnumValue(key, i) + except OSError: + break + if v and vt == winreg.REG_SZ and os.path.isdir(vc_dir): + try: + version = int(float(v)) + except (ValueError, TypeError): + continue + if version >= 14 and version > best_version: + best_version, best_dir = version, vc_dir + return best_version, best_dir + +def _find_vc2017(): + """Returns "15, path" based on the result of invoking vswhere.exe + If no install is found, returns "None, None" + + The version is returned to avoid unnecessarily changing the function + result. It may be ignored when the path is not None. + + If vswhere.exe is not available, by definition, VS 2017 is not + installed. + """ + root = os.environ.get("ProgramFiles(x86)") or os.environ.get("ProgramFiles") + if not root: + return None, None + + try: + path = subprocess.check_output([ + os.path.join(root, "Microsoft Visual Studio", "Installer", "vswhere.exe"), + "-latest", + "-prerelease", + "-requires", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", + "-property", "installationPath", + "-products", "*", + ], encoding="mbcs", errors="strict").strip() + except (subprocess.CalledProcessError, OSError, UnicodeDecodeError): + return None, None + + path = os.path.join(path, "VC", "Auxiliary", "Build") + if os.path.isdir(path): + return 15, path + + return None, None + +PLAT_SPEC_TO_RUNTIME = { + 'x86' : 'x86', + 'x86_amd64' : 'x64', + 'x86_arm' : 'arm', + 'x86_arm64' : 'arm64' +} + +def _find_vcvarsall(plat_spec): + # bpo-38597: Removed vcruntime return value + _, best_dir = _find_vc2017() + + if not best_dir: + best_version, best_dir = _find_vc2015() + + if not best_dir: + log.debug("No suitable Visual C++ version found") + return None, None + + vcvarsall = os.path.join(best_dir, "vcvarsall.bat") + if not os.path.isfile(vcvarsall): + log.debug("%s cannot be found", vcvarsall) + return None, None + + return vcvarsall, None + +def _get_vc_env(plat_spec): + if os.getenv("DISTUTILS_USE_SDK"): + return { + key.lower(): value + for key, value in os.environ.items() + } + + vcvarsall, _ = _find_vcvarsall(plat_spec) + if not vcvarsall: + raise DistutilsPlatformError("Unable to find vcvarsall.bat") + + try: + out = subprocess.check_output( + 'cmd /u /c "{}" {} && set'.format(vcvarsall, plat_spec), + stderr=subprocess.STDOUT, + ).decode('utf-16le', errors='replace') + except subprocess.CalledProcessError as exc: + log.error(exc.output) + raise DistutilsPlatformError("Error executing {}" + .format(exc.cmd)) + + env = { + key.lower(): value + for key, _, value in + (line.partition('=') for line in out.splitlines()) + if key and value + } + + return env + +def _find_exe(exe, paths=None): + """Return path to an MSVC executable program. + + Tries to find the program in several places: first, one of the + MSVC program search paths from the registry; next, the directories + in the PATH environment variable. If any of those work, return an + absolute path that is known to exist. If none of them work, just + return the original program name, 'exe'. + """ + if not paths: + paths = os.getenv('path').split(os.pathsep) + for p in paths: + fn = os.path.join(os.path.abspath(p), exe) + if os.path.isfile(fn): + return fn + return exe + +# A map keyed by get_platform() return values to values accepted by +# 'vcvarsall.bat'. Always cross-compile from x86 to work with the +# lighter-weight MSVC installs that do not include native 64-bit tools. +PLAT_TO_VCVARS = { + 'win32' : 'x86', + 'win-amd64' : 'x86_amd64', + 'win-arm32' : 'x86_arm', + 'win-arm64' : 'x86_arm64' +} + +class MSVCCompiler(CCompiler) : + """Concrete class that implements an interface to Microsoft Visual C++, + as defined by the CCompiler abstract class.""" + + compiler_type = 'msvc' + + # Just set this so CCompiler's constructor doesn't barf. We currently + # don't use the 'set_executables()' bureaucracy provided by CCompiler, + # as it really isn't necessary for this sort of single-compiler class. + # Would be nice to have a consistent interface with UnixCCompiler, + # though, so it's worth thinking about. + executables = {} + + # Private class data (need to distinguish C from C++ source for compiler) + _c_extensions = ['.c'] + _cpp_extensions = ['.cc', '.cpp', '.cxx'] + _rc_extensions = ['.rc'] + _mc_extensions = ['.mc'] + + # Needed for the filename generation methods provided by the + # base class, CCompiler. + src_extensions = (_c_extensions + _cpp_extensions + + _rc_extensions + _mc_extensions) + res_extension = '.res' + obj_extension = '.obj' + static_lib_extension = '.lib' + shared_lib_extension = '.dll' + static_lib_format = shared_lib_format = '%s%s' + exe_extension = '.exe' + + + def __init__(self, verbose=0, dry_run=0, force=0): + CCompiler.__init__ (self, verbose, dry_run, force) + # target platform (.plat_name is consistent with 'bdist') + self.plat_name = None + self.initialized = False diff --git a/Tools/c-analyzer/distutils/bcppcompiler.py b/Tools/c-analyzer/distutils/bcppcompiler.py new file mode 100644 index 000000000000000..4575b665c10c0e5 --- /dev/null +++ b/Tools/c-analyzer/distutils/bcppcompiler.py @@ -0,0 +1,109 @@ +"""distutils.bcppcompiler + +Contains BorlandCCompiler, an implementation of the abstract CCompiler class +for the Borland C++ compiler. +""" + +# This implementation by Lyle Johnson, based on the original msvccompiler.py +# module and using the directions originally published by Gordon Williams. + +# XXX looks like there's a LOT of overlap between these two classes: +# someone should sit down and factor out the common code as +# WindowsCCompiler! --GPW + + +import os +from distutils.errors import DistutilsExecError, CompileError +from distutils.ccompiler import \ + CCompiler, gen_preprocess_options +from distutils.dep_util import newer + +class BCPPCompiler(CCompiler) : + """Concrete class that implements an interface to the Borland C/C++ + compiler, as defined by the CCompiler abstract class. + """ + + compiler_type = 'bcpp' + + # Just set this so CCompiler's constructor doesn't barf. We currently + # don't use the 'set_executables()' bureaucracy provided by CCompiler, + # as it really isn't necessary for this sort of single-compiler class. + # Would be nice to have a consistent interface with UnixCCompiler, + # though, so it's worth thinking about. + executables = {} + + # Private class data (need to distinguish C from C++ source for compiler) + _c_extensions = ['.c'] + _cpp_extensions = ['.cc', '.cpp', '.cxx'] + + # Needed for the filename generation methods provided by the + # base class, CCompiler. + src_extensions = _c_extensions + _cpp_extensions + obj_extension = '.obj' + static_lib_extension = '.lib' + shared_lib_extension = '.dll' + static_lib_format = shared_lib_format = '%s%s' + exe_extension = '.exe' + + + def __init__ (self, + verbose=0, + dry_run=0, + force=0): + + CCompiler.__init__ (self, verbose, dry_run, force) + + # These executables are assumed to all be in the path. + # Borland doesn't seem to use any special registry settings to + # indicate their installation locations. + + self.cc = "bcc32.exe" + self.linker = "ilink32.exe" + self.lib = "tlib.exe" + + self.preprocess_options = None + self.compile_options = ['/tWM', '/O2', '/q', '/g0'] + self.compile_options_debug = ['/tWM', '/Od', '/q', '/g0'] + + self.ldflags_shared = ['/Tpd', '/Gn', '/q', '/x'] + self.ldflags_shared_debug = ['/Tpd', '/Gn', '/q', '/x'] + self.ldflags_static = [] + self.ldflags_exe = ['/Gn', '/q', '/x'] + self.ldflags_exe_debug = ['/Gn', '/q', '/x','/r'] + + + # -- Worker methods ------------------------------------------------ + + def preprocess (self, + source, + output_file=None, + macros=None, + include_dirs=None, + extra_preargs=None, + extra_postargs=None): + + (_, macros, include_dirs) = \ + self._fix_compile_args(None, macros, include_dirs) + pp_opts = gen_preprocess_options(macros, include_dirs) + pp_args = ['cpp32.exe'] + pp_opts + if output_file is not None: + pp_args.append('-o' + output_file) + if extra_preargs: + pp_args[:0] = extra_preargs + if extra_postargs: + pp_args.extend(extra_postargs) + pp_args.append(source) + + # We need to preprocess: either we're being forced to, or the + # source file is newer than the target (or the target doesn't + # exist). + if self.force or output_file is None or newer(source, output_file): + if output_file: + self.mkpath(os.path.dirname(output_file)) + try: + self.spawn(pp_args) + except DistutilsExecError as msg: + print(msg) + raise CompileError(msg) + + # preprocess() diff --git a/Tools/c-analyzer/distutils/ccompiler.py b/Tools/c-analyzer/distutils/ccompiler.py new file mode 100644 index 000000000000000..13e43103b94f5eb --- /dev/null +++ b/Tools/c-analyzer/distutils/ccompiler.py @@ -0,0 +1,470 @@ +"""distutils.ccompiler + +Contains CCompiler, an abstract base class that defines the interface +for the Distutils compiler abstraction model.""" + +import sys, os, re +from distutils.errors import ( + DistutilsModuleError, DistutilsPlatformError, +) +from distutils.util import split_quoted + +class CCompiler: + """Abstract base class to define the interface that must be implemented + by real compiler classes. Also has some utility methods used by + several compiler classes. + + The basic idea behind a compiler abstraction class is that each + instance can be used for all the compile/link steps in building a + single project. Thus, attributes common to all of those compile and + link steps -- include directories, macros to define, libraries to link + against, etc. -- are attributes of the compiler instance. To allow for + variability in how individual files are treated, most of those + attributes may be varied on a per-compilation or per-link basis. + """ + + # 'compiler_type' is a class attribute that identifies this class. It + # keeps code that wants to know what kind of compiler it's dealing with + # from having to import all possible compiler classes just to do an + # 'isinstance'. In concrete CCompiler subclasses, 'compiler_type' + # should really, really be one of the keys of the 'compiler_class' + # dictionary (see below -- used by the 'new_compiler()' factory + # function) -- authors of new compiler interface classes are + # responsible for updating 'compiler_class'! + compiler_type = None + + # XXX things not handled by this compiler abstraction model: + # * client can't provide additional options for a compiler, + # e.g. warning, optimization, debugging flags. Perhaps this + # should be the domain of concrete compiler abstraction classes + # (UnixCCompiler, MSVCCompiler, etc.) -- or perhaps the base + # class should have methods for the common ones. + # * can't completely override the include or library searchg + # path, ie. no "cc -I -Idir1 -Idir2" or "cc -L -Ldir1 -Ldir2". + # I'm not sure how widely supported this is even by Unix + # compilers, much less on other platforms. And I'm even less + # sure how useful it is; maybe for cross-compiling, but + # support for that is a ways off. (And anyways, cross + # compilers probably have a dedicated binary with the + # right paths compiled in. I hope.) + # * can't do really freaky things with the library list/library + # dirs, e.g. "-Ldir1 -lfoo -Ldir2 -lfoo" to link against + # different versions of libfoo.a in different locations. I + # think this is useless without the ability to null out the + # library search path anyways. + + + # Subclasses that rely on the standard filename generation methods + # implemented below should override these; see the comment near + # those methods ('object_filenames()' et. al.) for details: + src_extensions = None # list of strings + obj_extension = None # string + static_lib_extension = None + shared_lib_extension = None # string + static_lib_format = None # format string + shared_lib_format = None # prob. same as static_lib_format + exe_extension = None # string + + # Default language settings. language_map is used to detect a source + # file or Extension target language, checking source filenames. + # language_order is used to detect the language precedence, when deciding + # what language to use when mixing source types. For example, if some + # extension has two files with ".c" extension, and one with ".cpp", it + # is still linked as c++. + language_map = {".c" : "c", + ".cc" : "c++", + ".cpp" : "c++", + ".cxx" : "c++", + ".m" : "objc", + } + language_order = ["c++", "objc", "c"] + + def __init__(self, verbose=0, dry_run=0, force=0): + self.dry_run = dry_run + self.force = force + self.verbose = verbose + + # 'output_dir': a common output directory for object, library, + # shared object, and shared library files + self.output_dir = None + + # 'macros': a list of macro definitions (or undefinitions). A + # macro definition is a 2-tuple (name, value), where the value is + # either a string or None (no explicit value). A macro + # undefinition is a 1-tuple (name,). + self.macros = [] + + # 'include_dirs': a list of directories to search for include files + self.include_dirs = [] + + # 'libraries': a list of libraries to include in any link + # (library names, not filenames: eg. "foo" not "libfoo.a") + self.libraries = [] + + # 'library_dirs': a list of directories to search for libraries + self.library_dirs = [] + + # 'runtime_library_dirs': a list of directories to search for + # shared libraries/objects at runtime + self.runtime_library_dirs = [] + + # 'objects': a list of object files (or similar, such as explicitly + # named library files) to include on any link + self.objects = [] + + for key in self.executables.keys(): + self.set_executable(key, self.executables[key]) + + def set_executables(self, **kwargs): + """Define the executables (and options for them) that will be run + to perform the various stages of compilation. The exact set of + executables that may be specified here depends on the compiler + class (via the 'executables' class attribute), but most will have: + compiler the C/C++ compiler + linker_so linker used to create shared objects and libraries + linker_exe linker used to create binary executables + archiver static library creator + + On platforms with a command-line (Unix, DOS/Windows), each of these + is a string that will be split into executable name and (optional) + list of arguments. (Splitting the string is done similarly to how + Unix shells operate: words are delimited by spaces, but quotes and + backslashes can override this. See + 'distutils.util.split_quoted()'.) + """ + + # Note that some CCompiler implementation classes will define class + # attributes 'cpp', 'cc', etc. with hard-coded executable names; + # this is appropriate when a compiler class is for exactly one + # compiler/OS combination (eg. MSVCCompiler). Other compiler + # classes (UnixCCompiler, in particular) are driven by information + # discovered at run-time, since there are many different ways to do + # basically the same things with Unix C compilers. + + for key in kwargs: + if key not in self.executables: + raise ValueError("unknown executable '%s' for class %s" % + (key, self.__class__.__name__)) + self.set_executable(key, kwargs[key]) + + def set_executable(self, key, value): + if isinstance(value, str): + setattr(self, key, split_quoted(value)) + else: + setattr(self, key, value) + + def _find_macro(self, name): + i = 0 + for defn in self.macros: + if defn[0] == name: + return i + i += 1 + return None + + def _check_macro_definitions(self, definitions): + """Ensures that every element of 'definitions' is a valid macro + definition, ie. either (name,value) 2-tuple or a (name,) tuple. Do + nothing if all definitions are OK, raise TypeError otherwise. + """ + for defn in definitions: + if not (isinstance(defn, tuple) and + (len(defn) in (1, 2) and + (isinstance (defn[1], str) or defn[1] is None)) and + isinstance (defn[0], str)): + raise TypeError(("invalid macro definition '%s': " % defn) + \ + "must be tuple (string,), (string, string), or " + \ + "(string, None)") + + + # -- Bookkeeping methods ------------------------------------------- + + def define_macro(self, name, value=None): + """Define a preprocessor macro for all compilations driven by this + compiler object. The optional parameter 'value' should be a + string; if it is not supplied, then the macro will be defined + without an explicit value and the exact outcome depends on the + compiler used (XXX true? does ANSI say anything about this?) + """ + # Delete from the list of macro definitions/undefinitions if + # already there (so that this one will take precedence). + i = self._find_macro (name) + if i is not None: + del self.macros[i] + + self.macros.append((name, value)) + + def undefine_macro(self, name): + """Undefine a preprocessor macro for all compilations driven by + this compiler object. If the same macro is defined by + 'define_macro()' and undefined by 'undefine_macro()' the last call + takes precedence (including multiple redefinitions or + undefinitions). If the macro is redefined/undefined on a + per-compilation basis (ie. in the call to 'compile()'), then that + takes precedence. + """ + # Delete from the list of macro definitions/undefinitions if + # already there (so that this one will take precedence). + i = self._find_macro (name) + if i is not None: + del self.macros[i] + + undefn = (name,) + self.macros.append(undefn) + + def add_include_dir(self, dir): + """Add 'dir' to the list of directories that will be searched for + header files. The compiler is instructed to search directories in + the order in which they are supplied by successive calls to + 'add_include_dir()'. + """ + self.include_dirs.append(dir) + + def set_include_dirs(self, dirs): + """Set the list of directories that will be searched to 'dirs' (a + list of strings). Overrides any preceding calls to + 'add_include_dir()'; subsequence calls to 'add_include_dir()' add + to the list passed to 'set_include_dirs()'. This does not affect + any list of standard include directories that the compiler may + search by default. + """ + self.include_dirs = dirs[:] + + + # -- Private utility methods -------------------------------------- + # (here for the convenience of subclasses) + + # Helper method to prep compiler in subclass compile() methods + + def _fix_compile_args(self, output_dir, macros, include_dirs): + """Typecheck and fix-up some of the arguments to the 'compile()' + method, and return fixed-up values. Specifically: if 'output_dir' + is None, replaces it with 'self.output_dir'; ensures that 'macros' + is a list, and augments it with 'self.macros'; ensures that + 'include_dirs' is a list, and augments it with 'self.include_dirs'. + Guarantees that the returned values are of the correct type, + i.e. for 'output_dir' either string or None, and for 'macros' and + 'include_dirs' either list or None. + """ + if output_dir is None: + output_dir = self.output_dir + elif not isinstance(output_dir, str): + raise TypeError("'output_dir' must be a string or None") + + if macros is None: + macros = self.macros + elif isinstance(macros, list): + macros = macros + (self.macros or []) + else: + raise TypeError("'macros' (if supplied) must be a list of tuples") + + if include_dirs is None: + include_dirs = self.include_dirs + elif isinstance(include_dirs, (list, tuple)): + include_dirs = list(include_dirs) + (self.include_dirs or []) + else: + raise TypeError( + "'include_dirs' (if supplied) must be a list of strings") + + return output_dir, macros, include_dirs + + + # -- Worker methods ------------------------------------------------ + # (must be implemented by subclasses) + + def preprocess(self, source, output_file=None, macros=None, + include_dirs=None, extra_preargs=None, extra_postargs=None): + """Preprocess a single C/C++ source file, named in 'source'. + Output will be written to file named 'output_file', or stdout if + 'output_file' not supplied. 'macros' is a list of macro + definitions as for 'compile()', which will augment the macros set + with 'define_macro()' and 'undefine_macro()'. 'include_dirs' is a + list of directory names that will be added to the default list. + + Raises PreprocessError on failure. + """ + pass + + + # -- Miscellaneous methods ----------------------------------------- + # These are all used by the 'gen_lib_options() function; there is + # no appropriate default implementation so subclasses should + # implement all of these. + +# def library_dir_option(self, dir): +# """Return the compiler option to add 'dir' to the list of +# directories searched for libraries. +# """ +# raise NotImplementedError +# +# def runtime_library_dir_option(self, dir): +# """Return the compiler option to add 'dir' to the list of +# directories searched for runtime libraries. +# """ +# raise NotImplementedError +# +# def library_option(self, lib): +# """Return the compiler option to add 'lib' to the list of libraries +# linked into the shared library or executable. +# """ +# raise NotImplementedError +# +# def find_library_file (self, dirs, lib, debug=0): +# """Search the specified list of directories for a static or shared +# library file 'lib' and return the full path to that file. If +# 'debug' true, look for a debugging version (if that makes sense on +# the current platform). Return None if 'lib' wasn't found in any of +# the specified directories. +# """ +# raise NotImplementedError + + + # -- Utility methods ----------------------------------------------- + + def spawn(self, cmd): + raise NotImplementedError + + +# Map a sys.platform/os.name ('posix', 'nt') to the default compiler +# type for that platform. Keys are interpreted as re match +# patterns. Order is important; platform mappings are preferred over +# OS names. +_default_compilers = ( + + # Platform string mappings + + # on a cygwin built python we can use gcc like an ordinary UNIXish + # compiler + ('cygwin.*', 'unix'), + + # OS name mappings + ('posix', 'unix'), + ('nt', 'msvc'), + + ) + +def get_default_compiler(osname=None, platform=None): + """Determine the default compiler to use for the given platform. + + osname should be one of the standard Python OS names (i.e. the + ones returned by os.name) and platform the common value + returned by sys.platform for the platform in question. + + The default values are os.name and sys.platform in case the + parameters are not given. + """ + if osname is None: + osname = os.name + if platform is None: + platform = sys.platform + for pattern, compiler in _default_compilers: + if re.match(pattern, platform) is not None or \ + re.match(pattern, osname) is not None: + return compiler + # Default to Unix compiler + return 'unix' + +# Map compiler types to (module_name, class_name) pairs -- ie. where to +# find the code that implements an interface to this compiler. (The module +# is assumed to be in the 'distutils' package.) +compiler_class = { 'unix': ('unixccompiler', 'UnixCCompiler', + "standard UNIX-style compiler"), + 'msvc': ('_msvccompiler', 'MSVCCompiler', + "Microsoft Visual C++"), + 'cygwin': ('cygwinccompiler', 'CygwinCCompiler', + "Cygwin port of GNU C Compiler for Win32"), + 'mingw32': ('cygwinccompiler', 'Mingw32CCompiler', + "Mingw32 port of GNU C Compiler for Win32"), + 'bcpp': ('bcppcompiler', 'BCPPCompiler', + "Borland C++ Compiler"), + } + + +def new_compiler(plat=None, compiler=None, verbose=0, dry_run=0, force=0): + """Generate an instance of some CCompiler subclass for the supplied + platform/compiler combination. 'plat' defaults to 'os.name' + (eg. 'posix', 'nt'), and 'compiler' defaults to the default compiler + for that platform. Currently only 'posix' and 'nt' are supported, and + the default compilers are "traditional Unix interface" (UnixCCompiler + class) and Visual C++ (MSVCCompiler class). Note that it's perfectly + possible to ask for a Unix compiler object under Windows, and a + Microsoft compiler object under Unix -- if you supply a value for + 'compiler', 'plat' is ignored. + """ + if plat is None: + plat = os.name + + try: + if compiler is None: + compiler = get_default_compiler(plat) + + (module_name, class_name, long_description) = compiler_class[compiler] + except KeyError: + msg = "don't know how to compile C/C++ code on platform '%s'" % plat + if compiler is not None: + msg = msg + " with '%s' compiler" % compiler + raise DistutilsPlatformError(msg) + + try: + module_name = "distutils." + module_name + __import__ (module_name) + module = sys.modules[module_name] + klass = vars(module)[class_name] + except ImportError: + raise + raise DistutilsModuleError( + "can't compile C/C++ code: unable to load module '%s'" % \ + module_name) + except KeyError: + raise DistutilsModuleError( + "can't compile C/C++ code: unable to find class '%s' " + "in module '%s'" % (class_name, module_name)) + + # XXX The None is necessary to preserve backwards compatibility + # with classes that expect verbose to be the first positional + # argument. + return klass(None, dry_run, force) + + +def gen_preprocess_options(macros, include_dirs): + """Generate C pre-processor options (-D, -U, -I) as used by at least + two types of compilers: the typical Unix compiler and Visual C++. + 'macros' is the usual thing, a list of 1- or 2-tuples, where (name,) + means undefine (-U) macro 'name', and (name,value) means define (-D) + macro 'name' to 'value'. 'include_dirs' is just a list of directory + names to be added to the header file search path (-I). Returns a list + of command-line options suitable for either Unix compilers or Visual + C++. + """ + # XXX it would be nice (mainly aesthetic, and so we don't generate + # stupid-looking command lines) to go over 'macros' and eliminate + # redundant definitions/undefinitions (ie. ensure that only the + # latest mention of a particular macro winds up on the command + # line). I don't think it's essential, though, since most (all?) + # Unix C compilers only pay attention to the latest -D or -U + # mention of a macro on their command line. Similar situation for + # 'include_dirs'. I'm punting on both for now. Anyways, weeding out + # redundancies like this should probably be the province of + # CCompiler, since the data structures used are inherited from it + # and therefore common to all CCompiler classes. + pp_opts = [] + for macro in macros: + if not (isinstance(macro, tuple) and 1 <= len(macro) <= 2): + raise TypeError( + "bad macro definition '%s': " + "each element of 'macros' list must be a 1- or 2-tuple" + % macro) + + if len(macro) == 1: # undefine this macro + pp_opts.append("-U%s" % macro[0]) + elif len(macro) == 2: + if macro[1] is None: # define with no explicit value + pp_opts.append("-D%s" % macro[0]) + else: + # XXX *don't* need to be clever about quoting the + # macro value here, because we're going to avoid the + # shell at all costs when we spawn the command! + pp_opts.append("-D%s=%s" % macro) + + for dir in include_dirs: + pp_opts.append("-I%s" % dir) + return pp_opts diff --git a/Tools/c-analyzer/distutils/cygwinccompiler.py b/Tools/c-analyzer/distutils/cygwinccompiler.py new file mode 100644 index 000000000000000..a3505f31f1f0a26 --- /dev/null +++ b/Tools/c-analyzer/distutils/cygwinccompiler.py @@ -0,0 +1,286 @@ +"""distutils.cygwinccompiler + +Provides the CygwinCCompiler class, a subclass of UnixCCompiler that +handles the Cygwin port of the GNU C compiler to Windows. It also contains +the Mingw32CCompiler class which handles the mingw32 port of GCC (same as +cygwin in no-cygwin mode). +""" + +# problems: +# +# * if you use a msvc compiled python version (1.5.2) +# 1. you have to insert a __GNUC__ section in its config.h +# 2. you have to generate an import library for its dll +# - create a def-file for python??.dll +# - create an import library using +# dlltool --dllname python15.dll --def python15.def \ +# --output-lib libpython15.a +# +# see also http://starship.python.net/crew/kernr/mingw32/Notes.html +# +# * We put export_symbols in a def-file, and don't use +# --export-all-symbols because it doesn't worked reliable in some +# tested configurations. And because other windows compilers also +# need their symbols specified this no serious problem. +# +# tested configurations: +# +# * cygwin gcc 2.91.57/ld 2.9.4/dllwrap 0.2.4 works +# (after patching python's config.h and for C++ some other include files) +# see also http://starship.python.net/crew/kernr/mingw32/Notes.html +# * mingw32 gcc 2.95.2/ld 2.9.4/dllwrap 0.2.4 works +# (ld doesn't support -shared, so we use dllwrap) +# * cygwin gcc 2.95.2/ld 2.10.90/dllwrap 2.10.90 works now +# - its dllwrap doesn't work, there is a bug in binutils 2.10.90 +# see also http://sources.redhat.com/ml/cygwin/2000-06/msg01274.html +# - using gcc -mdll instead dllwrap doesn't work without -static because +# it tries to link against dlls instead their import libraries. (If +# it finds the dll first.) +# By specifying -static we force ld to link against the import libraries, +# this is windows standard and there are normally not the necessary symbols +# in the dlls. +# *** only the version of June 2000 shows these problems +# * cygwin gcc 3.2/ld 2.13.90 works +# (ld supports -shared) +# * mingw gcc 3.2/ld 2.13 works +# (ld supports -shared) + +import os +import sys +from subprocess import Popen, PIPE, check_output +import re + +from distutils.unixccompiler import UnixCCompiler +from distutils.errors import CCompilerError +from distutils.version import LooseVersion +from distutils.spawn import find_executable + +def get_msvcr(): + """Include the appropriate MSVC runtime library if Python was built + with MSVC 7.0 or later. + """ + msc_pos = sys.version.find('MSC v.') + if msc_pos != -1: + msc_ver = sys.version[msc_pos+6:msc_pos+10] + if msc_ver == '1300': + # MSVC 7.0 + return ['msvcr70'] + elif msc_ver == '1310': + # MSVC 7.1 + return ['msvcr71'] + elif msc_ver == '1400': + # VS2005 / MSVC 8.0 + return ['msvcr80'] + elif msc_ver == '1500': + # VS2008 / MSVC 9.0 + return ['msvcr90'] + elif msc_ver == '1600': + # VS2010 / MSVC 10.0 + return ['msvcr100'] + else: + raise ValueError("Unknown MS Compiler version %s " % msc_ver) + + +class CygwinCCompiler(UnixCCompiler): + """ Handles the Cygwin port of the GNU C compiler to Windows. + """ + compiler_type = 'cygwin' + obj_extension = ".o" + static_lib_extension = ".a" + shared_lib_extension = ".dll" + static_lib_format = "lib%s%s" + shared_lib_format = "%s%s" + exe_extension = ".exe" + + def __init__(self, verbose=0, dry_run=0, force=0): + + UnixCCompiler.__init__(self, verbose, dry_run, force) + + status, details = check_config_h() + self.debug_print("Python's GCC status: %s (details: %s)" % + (status, details)) + if status is not CONFIG_H_OK: + self.warn( + "Python's pyconfig.h doesn't seem to support your compiler. " + "Reason: %s. " + "Compiling may fail because of undefined preprocessor macros." + % details) + + self.gcc_version, self.ld_version, self.dllwrap_version = \ + get_versions() + self.debug_print(self.compiler_type + ": gcc %s, ld %s, dllwrap %s\n" % + (self.gcc_version, + self.ld_version, + self.dllwrap_version) ) + + # ld_version >= "2.10.90" and < "2.13" should also be able to use + # gcc -mdll instead of dllwrap + # Older dllwraps had own version numbers, newer ones use the + # same as the rest of binutils ( also ld ) + # dllwrap 2.10.90 is buggy + if self.ld_version >= "2.10.90": + self.linker_dll = "gcc" + else: + self.linker_dll = "dllwrap" + + # ld_version >= "2.13" support -shared so use it instead of + # -mdll -static + if self.ld_version >= "2.13": + shared_option = "-shared" + else: + shared_option = "-mdll -static" + + # Hard-code GCC because that's what this is all about. + # XXX optimization, warnings etc. should be customizable. + self.set_executables(compiler='gcc -mcygwin -O -Wall', + compiler_so='gcc -mcygwin -mdll -O -Wall', + compiler_cxx='g++ -mcygwin -O -Wall', + linker_exe='gcc -mcygwin', + linker_so=('%s -mcygwin %s' % + (self.linker_dll, shared_option))) + + # cygwin and mingw32 need different sets of libraries + if self.gcc_version == "2.91.57": + # cygwin shouldn't need msvcrt, but without the dlls will crash + # (gcc version 2.91.57) -- perhaps something about initialization + self.dll_libraries=["msvcrt"] + self.warn( + "Consider upgrading to a newer version of gcc") + else: + # Include the appropriate MSVC runtime library if Python was built + # with MSVC 7.0 or later. + self.dll_libraries = get_msvcr() + + +# the same as cygwin plus some additional parameters +class Mingw32CCompiler(CygwinCCompiler): + """ Handles the Mingw32 port of the GNU C compiler to Windows. + """ + compiler_type = 'mingw32' + + def __init__(self, verbose=0, dry_run=0, force=0): + + CygwinCCompiler.__init__ (self, verbose, dry_run, force) + + # ld_version >= "2.13" support -shared so use it instead of + # -mdll -static + if self.ld_version >= "2.13": + shared_option = "-shared" + else: + shared_option = "-mdll -static" + + # A real mingw32 doesn't need to specify a different entry point, + # but cygwin 2.91.57 in no-cygwin-mode needs it. + if self.gcc_version <= "2.91.57": + entry_point = '--entry _DllMain@12' + else: + entry_point = '' + + if is_cygwingcc(): + raise CCompilerError( + 'Cygwin gcc cannot be used with --compiler=mingw32') + + self.set_executables(compiler='gcc -O -Wall', + compiler_so='gcc -mdll -O -Wall', + compiler_cxx='g++ -O -Wall', + linker_exe='gcc', + linker_so='%s %s %s' + % (self.linker_dll, shared_option, + entry_point)) + # Maybe we should also append -mthreads, but then the finished + # dlls need another dll (mingwm10.dll see Mingw32 docs) + # (-mthreads: Support thread-safe exception handling on `Mingw32') + + # no additional libraries needed + self.dll_libraries=[] + + # Include the appropriate MSVC runtime library if Python was built + # with MSVC 7.0 or later. + self.dll_libraries = get_msvcr() + +# Because these compilers aren't configured in Python's pyconfig.h file by +# default, we should at least warn the user if he is using an unmodified +# version. + +CONFIG_H_OK = "ok" +CONFIG_H_NOTOK = "not ok" +CONFIG_H_UNCERTAIN = "uncertain" + +def check_config_h(): + """Check if the current Python installation appears amenable to building + extensions with GCC. + + Returns a tuple (status, details), where 'status' is one of the following + constants: + + - CONFIG_H_OK: all is well, go ahead and compile + - CONFIG_H_NOTOK: doesn't look good + - CONFIG_H_UNCERTAIN: not sure -- unable to read pyconfig.h + + 'details' is a human-readable string explaining the situation. + + Note there are two ways to conclude "OK": either 'sys.version' contains + the string "GCC" (implying that this Python was built with GCC), or the + installed "pyconfig.h" contains the string "__GNUC__". + """ + + # XXX since this function also checks sys.version, it's not strictly a + # "pyconfig.h" check -- should probably be renamed... + + import sysconfig + + # if sys.version contains GCC then python was compiled with GCC, and the + # pyconfig.h file should be OK + if "GCC" in sys.version: + return CONFIG_H_OK, "sys.version mentions 'GCC'" + + # let's see if __GNUC__ is mentioned in python.h + fn = sysconfig.get_config_h_filename() + try: + config_h = open(fn) + try: + if "__GNUC__" in config_h.read(): + return CONFIG_H_OK, "'%s' mentions '__GNUC__'" % fn + else: + return CONFIG_H_NOTOK, "'%s' does not mention '__GNUC__'" % fn + finally: + config_h.close() + except OSError as exc: + return (CONFIG_H_UNCERTAIN, + "couldn't read '%s': %s" % (fn, exc.strerror)) + +RE_VERSION = re.compile(br'(\d+\.\d+(\.\d+)*)') + +def _find_exe_version(cmd): + """Find the version of an executable by running `cmd` in the shell. + + If the command is not found, or the output does not match + `RE_VERSION`, returns None. + """ + executable = cmd.split()[0] + if find_executable(executable) is None: + return None + out = Popen(cmd, shell=True, stdout=PIPE).stdout + try: + out_string = out.read() + finally: + out.close() + result = RE_VERSION.search(out_string) + if result is None: + return None + # LooseVersion works with strings + # so we need to decode our bytes + return LooseVersion(result.group(1).decode()) + +def get_versions(): + """ Try to find out the versions of gcc, ld and dllwrap. + + If not possible it returns None for it. + """ + commands = ['gcc -dumpversion', 'ld -v', 'dllwrap --version'] + return tuple([_find_exe_version(cmd) for cmd in commands]) + +def is_cygwingcc(): + '''Try to determine if the gcc that would be used is from cygwin.''' + out_string = check_output(['gcc', '-dumpmachine']) + return out_string.strip().endswith(b'cygwin') diff --git a/Tools/c-analyzer/distutils/debug.py b/Tools/c-analyzer/distutils/debug.py new file mode 100644 index 000000000000000..daf1660f0d82114 --- /dev/null +++ b/Tools/c-analyzer/distutils/debug.py @@ -0,0 +1,5 @@ +import os + +# If DISTUTILS_DEBUG is anything other than the empty string, we run in +# debug mode. +DEBUG = os.environ.get('DISTUTILS_DEBUG') diff --git a/Tools/c-analyzer/distutils/dep_util.py b/Tools/c-analyzer/distutils/dep_util.py new file mode 100644 index 000000000000000..318c830f2eab3e3 --- /dev/null +++ b/Tools/c-analyzer/distutils/dep_util.py @@ -0,0 +1,29 @@ +"""distutils.dep_util + +Utility functions for simple, timestamp-based dependency of files +and groups of files; also, function based entirely on such +timestamp dependency analysis.""" + +import os +from distutils.errors import DistutilsFileError + + +def newer (source, target): + """Return true if 'source' exists and is more recently modified than + 'target', or if 'source' exists and 'target' doesn't. Return false if + both exist and 'target' is the same age or younger than 'source'. + Raise DistutilsFileError if 'source' does not exist. + """ + if not os.path.exists(source): + raise DistutilsFileError("file '%s' does not exist" % + os.path.abspath(source)) + if not os.path.exists(target): + return 1 + + from stat import ST_MTIME + mtime1 = os.stat(source)[ST_MTIME] + mtime2 = os.stat(target)[ST_MTIME] + + return mtime1 > mtime2 + +# newer () diff --git a/Tools/c-analyzer/distutils/errors.py b/Tools/c-analyzer/distutils/errors.py new file mode 100644 index 000000000000000..10f8e316f675f08 --- /dev/null +++ b/Tools/c-analyzer/distutils/errors.py @@ -0,0 +1,48 @@ +"""distutils.errors + +Provides exceptions used by the Distutils modules. Note that Distutils +modules may raise standard exceptions; in particular, SystemExit is +usually raised for errors that are obviously the end-user's fault +(eg. bad command-line arguments). + +This module is safe to use in "from ... import *" mode; it only exports +symbols whose names start with "Distutils" and end with "Error".""" + +class DistutilsError (Exception): + """The root of all Distutils evil.""" + pass + +class DistutilsModuleError (DistutilsError): + """Unable to load an expected module, or to find an expected class + within some module (in particular, command modules and classes).""" + pass + +class DistutilsFileError (DistutilsError): + """Any problems in the filesystem: expected file not found, etc. + Typically this is for problems that we detect before OSError + could be raised.""" + pass + +class DistutilsPlatformError (DistutilsError): + """We don't know how to do something on the current platform (but + we do know how to do it on some platform) -- eg. trying to compile + C files on a platform not supported by a CCompiler subclass.""" + pass + +class DistutilsExecError (DistutilsError): + """Any problems executing an external program (such as the C + compiler, when compiling C files).""" + pass + +# Exception classes used by the CCompiler implementation classes +class CCompilerError (Exception): + """Some compile/link operation failed.""" + +class PreprocessError (CCompilerError): + """Failure to preprocess one or more C/C++ files.""" + +class CompileError (CCompilerError): + """Failure to compile one or more C/C++ source files.""" + +class UnknownFileError (CCompilerError): + """Attempt to process an unknown file type.""" diff --git a/Tools/c-analyzer/distutils/log.py b/Tools/c-analyzer/distutils/log.py new file mode 100644 index 000000000000000..26ecf22ae19befc --- /dev/null +++ b/Tools/c-analyzer/distutils/log.py @@ -0,0 +1,63 @@ +"""A simple log mechanism styled after PEP 282.""" + +# The class here is styled after PEP 282 so that it could later be +# replaced with a standard Python logging implementation. + +DEBUG = 1 +INFO = 2 +WARN = 3 +ERROR = 4 +FATAL = 5 + +import sys + +class Log: + + def __init__(self, threshold=WARN): + self.threshold = threshold + + def _log(self, level, msg, args): + if level not in (DEBUG, INFO, WARN, ERROR, FATAL): + raise ValueError('%s wrong log level' % str(level)) + + if level >= self.threshold: + if args: + msg = msg % args + if level in (WARN, ERROR, FATAL): + stream = sys.stderr + else: + stream = sys.stdout + try: + stream.write('%s\n' % msg) + except UnicodeEncodeError: + # emulate backslashreplace error handler + encoding = stream.encoding + msg = msg.encode(encoding, "backslashreplace").decode(encoding) + stream.write('%s\n' % msg) + stream.flush() + + def log(self, level, msg, *args): + self._log(level, msg, args) + + def debug(self, msg, *args): + self._log(DEBUG, msg, args) + + def info(self, msg, *args): + self._log(INFO, msg, args) + + def warn(self, msg, *args): + self._log(WARN, msg, args) + + def error(self, msg, *args): + self._log(ERROR, msg, args) + + def fatal(self, msg, *args): + self._log(FATAL, msg, args) + +_global_log = Log() +log = _global_log.log +debug = _global_log.debug +info = _global_log.info +warn = _global_log.warn +error = _global_log.error +fatal = _global_log.fatal diff --git a/Tools/c-analyzer/distutils/msvc9compiler.py b/Tools/c-analyzer/distutils/msvc9compiler.py new file mode 100644 index 000000000000000..38fff9b2d531209 --- /dev/null +++ b/Tools/c-analyzer/distutils/msvc9compiler.py @@ -0,0 +1,438 @@ +"""distutils.msvc9compiler + +Contains MSVCCompiler, an implementation of the abstract CCompiler class +for the Microsoft Visual Studio 2008. + +The module is compatible with VS 2005 and VS 2008. You can find legacy support +for older versions of VS in distutils.msvccompiler. +""" + +# Written by Perry Stoll +# hacked by Robin Becker and Thomas Heller to do a better job of +# finding DevStudio (through the registry) +# ported to VS2005 and VS 2008 by Christian Heimes + +import os +import subprocess +import sys +import re + +from distutils.errors import DistutilsPlatformError +from distutils.ccompiler import CCompiler +from distutils import log + +import winreg + +RegOpenKeyEx = winreg.OpenKeyEx +RegEnumKey = winreg.EnumKey +RegEnumValue = winreg.EnumValue +RegError = winreg.error + +HKEYS = (winreg.HKEY_USERS, + winreg.HKEY_CURRENT_USER, + winreg.HKEY_LOCAL_MACHINE, + winreg.HKEY_CLASSES_ROOT) + +NATIVE_WIN64 = (sys.platform == 'win32' and sys.maxsize > 2**32) +if NATIVE_WIN64: + # Visual C++ is a 32-bit application, so we need to look in + # the corresponding registry branch, if we're running a + # 64-bit Python on Win64 + VS_BASE = r"Software\Wow6432Node\Microsoft\VisualStudio\%0.1f" + WINSDK_BASE = r"Software\Wow6432Node\Microsoft\Microsoft SDKs\Windows" + NET_BASE = r"Software\Wow6432Node\Microsoft\.NETFramework" +else: + VS_BASE = r"Software\Microsoft\VisualStudio\%0.1f" + WINSDK_BASE = r"Software\Microsoft\Microsoft SDKs\Windows" + NET_BASE = r"Software\Microsoft\.NETFramework" + +# A map keyed by get_platform() return values to values accepted by +# 'vcvarsall.bat'. Note a cross-compile may combine these (eg, 'x86_amd64' is +# the param to cross-compile on x86 targeting amd64.) +PLAT_TO_VCVARS = { + 'win32' : 'x86', + 'win-amd64' : 'amd64', +} + +class Reg: + """Helper class to read values from the registry + """ + + def get_value(cls, path, key): + for base in HKEYS: + d = cls.read_values(base, path) + if d and key in d: + return d[key] + raise KeyError(key) + get_value = classmethod(get_value) + + def read_keys(cls, base, key): + """Return list of registry keys.""" + try: + handle = RegOpenKeyEx(base, key) + except RegError: + return None + L = [] + i = 0 + while True: + try: + k = RegEnumKey(handle, i) + except RegError: + break + L.append(k) + i += 1 + return L + read_keys = classmethod(read_keys) + + def read_values(cls, base, key): + """Return dict of registry keys and values. + + All names are converted to lowercase. + """ + try: + handle = RegOpenKeyEx(base, key) + except RegError: + return None + d = {} + i = 0 + while True: + try: + name, value, type = RegEnumValue(handle, i) + except RegError: + break + name = name.lower() + d[cls.convert_mbcs(name)] = cls.convert_mbcs(value) + i += 1 + return d + read_values = classmethod(read_values) + + def convert_mbcs(s): + dec = getattr(s, "decode", None) + if dec is not None: + try: + s = dec("mbcs") + except UnicodeError: + pass + return s + convert_mbcs = staticmethod(convert_mbcs) + +class MacroExpander: + + def __init__(self, version): + self.macros = {} + self.vsbase = VS_BASE % version + self.load_macros(version) + + def set_macro(self, macro, path, key): + self.macros["$(%s)" % macro] = Reg.get_value(path, key) + + def load_macros(self, version): + self.set_macro("VCInstallDir", self.vsbase + r"\Setup\VC", "productdir") + self.set_macro("VSInstallDir", self.vsbase + r"\Setup\VS", "productdir") + self.set_macro("FrameworkDir", NET_BASE, "installroot") + try: + if version >= 8.0: + self.set_macro("FrameworkSDKDir", NET_BASE, + "sdkinstallrootv2.0") + else: + raise KeyError("sdkinstallrootv2.0") + except KeyError: + raise DistutilsPlatformError( + """Python was built with Visual Studio 2008; +extensions must be built with a compiler than can generate compatible binaries. +Visual Studio 2008 was not found on this system. If you have Cygwin installed, +you can try compiling with MingW32, by passing "-c mingw32" to setup.py.""") + + if version >= 9.0: + self.set_macro("FrameworkVersion", self.vsbase, "clr version") + self.set_macro("WindowsSdkDir", WINSDK_BASE, "currentinstallfolder") + else: + p = r"Software\Microsoft\NET Framework Setup\Product" + for base in HKEYS: + try: + h = RegOpenKeyEx(base, p) + except RegError: + continue + key = RegEnumKey(h, 0) + d = Reg.get_value(base, r"%s\%s" % (p, key)) + self.macros["$(FrameworkVersion)"] = d["version"] + + def sub(self, s): + for k, v in self.macros.items(): + s = s.replace(k, v) + return s + +def get_build_version(): + """Return the version of MSVC that was used to build Python. + + For Python 2.3 and up, the version number is included in + sys.version. For earlier versions, assume the compiler is MSVC 6. + """ + prefix = "MSC v." + i = sys.version.find(prefix) + if i == -1: + return 6 + i = i + len(prefix) + s, rest = sys.version[i:].split(" ", 1) + majorVersion = int(s[:-2]) - 6 + if majorVersion >= 13: + # v13 was skipped and should be v14 + majorVersion += 1 + minorVersion = int(s[2:3]) / 10.0 + # I don't think paths are affected by minor version in version 6 + if majorVersion == 6: + minorVersion = 0 + if majorVersion >= 6: + return majorVersion + minorVersion + # else we don't know what version of the compiler this is + return None + +def normalize_and_reduce_paths(paths): + """Return a list of normalized paths with duplicates removed. + + The current order of paths is maintained. + """ + # Paths are normalized so things like: /a and /a/ aren't both preserved. + reduced_paths = [] + for p in paths: + np = os.path.normpath(p) + # XXX(nnorwitz): O(n**2), if reduced_paths gets long perhaps use a set. + if np not in reduced_paths: + reduced_paths.append(np) + return reduced_paths + +def removeDuplicates(variable): + """Remove duplicate values of an environment variable. + """ + oldList = variable.split(os.pathsep) + newList = [] + for i in oldList: + if i not in newList: + newList.append(i) + newVariable = os.pathsep.join(newList) + return newVariable + +def find_vcvarsall(version): + """Find the vcvarsall.bat file + + At first it tries to find the productdir of VS 2008 in the registry. If + that fails it falls back to the VS90COMNTOOLS env var. + """ + vsbase = VS_BASE % version + try: + productdir = Reg.get_value(r"%s\Setup\VC" % vsbase, + "productdir") + except KeyError: + log.debug("Unable to find productdir in registry") + productdir = None + + if not productdir or not os.path.isdir(productdir): + toolskey = "VS%0.f0COMNTOOLS" % version + toolsdir = os.environ.get(toolskey, None) + + if toolsdir and os.path.isdir(toolsdir): + productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC") + productdir = os.path.abspath(productdir) + if not os.path.isdir(productdir): + log.debug("%s is not a valid directory" % productdir) + return None + else: + log.debug("Env var %s is not set or invalid" % toolskey) + if not productdir: + log.debug("No productdir found") + return None + vcvarsall = os.path.join(productdir, "vcvarsall.bat") + if os.path.isfile(vcvarsall): + return vcvarsall + log.debug("Unable to find vcvarsall.bat") + return None + +def query_vcvarsall(version, arch="x86"): + """Launch vcvarsall.bat and read the settings from its environment + """ + vcvarsall = find_vcvarsall(version) + interesting = {"include", "lib", "libpath", "path"} + result = {} + + if vcvarsall is None: + raise DistutilsPlatformError("Unable to find vcvarsall.bat") + log.debug("Calling 'vcvarsall.bat %s' (version=%s)", arch, version) + popen = subprocess.Popen('"%s" %s & set' % (vcvarsall, arch), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + try: + stdout, stderr = popen.communicate() + if popen.wait() != 0: + raise DistutilsPlatformError(stderr.decode("mbcs")) + + stdout = stdout.decode("mbcs") + for line in stdout.split("\n"): + line = Reg.convert_mbcs(line) + if '=' not in line: + continue + line = line.strip() + key, value = line.split('=', 1) + key = key.lower() + if key in interesting: + if value.endswith(os.pathsep): + value = value[:-1] + result[key] = removeDuplicates(value) + + finally: + popen.stdout.close() + popen.stderr.close() + + if len(result) != len(interesting): + raise ValueError(str(list(result.keys()))) + + return result + +# More globals +VERSION = get_build_version() +if VERSION < 8.0: + raise DistutilsPlatformError("VC %0.1f is not supported by this module" % VERSION) +# MACROS = MacroExpander(VERSION) + +class MSVCCompiler(CCompiler) : + """Concrete class that implements an interface to Microsoft Visual C++, + as defined by the CCompiler abstract class.""" + + compiler_type = 'msvc' + + # Just set this so CCompiler's constructor doesn't barf. We currently + # don't use the 'set_executables()' bureaucracy provided by CCompiler, + # as it really isn't necessary for this sort of single-compiler class. + # Would be nice to have a consistent interface with UnixCCompiler, + # though, so it's worth thinking about. + executables = {} + + # Private class data (need to distinguish C from C++ source for compiler) + _c_extensions = ['.c'] + _cpp_extensions = ['.cc', '.cpp', '.cxx'] + _rc_extensions = ['.rc'] + _mc_extensions = ['.mc'] + + # Needed for the filename generation methods provided by the + # base class, CCompiler. + src_extensions = (_c_extensions + _cpp_extensions + + _rc_extensions + _mc_extensions) + res_extension = '.res' + obj_extension = '.obj' + static_lib_extension = '.lib' + shared_lib_extension = '.dll' + static_lib_format = shared_lib_format = '%s%s' + exe_extension = '.exe' + + def __init__(self, verbose=0, dry_run=0, force=0): + CCompiler.__init__ (self, verbose, dry_run, force) + self.__version = VERSION + self.__root = r"Software\Microsoft\VisualStudio" + # self.__macros = MACROS + self.__paths = [] + # target platform (.plat_name is consistent with 'bdist') + self.plat_name = None + self.__arch = None # deprecated name + self.initialized = False + + # -- Worker methods ------------------------------------------------ + + def manifest_setup_ldargs(self, output_filename, build_temp, ld_args): + # If we need a manifest at all, an embedded manifest is recommended. + # See MSDN article titled + # "How to: Embed a Manifest Inside a C/C++ Application" + # (currently at http://msdn2.microsoft.com/en-us/library/ms235591(VS.80).aspx) + # Ask the linker to generate the manifest in the temp dir, so + # we can check it, and possibly embed it, later. + temp_manifest = os.path.join( + build_temp, + os.path.basename(output_filename) + ".manifest") + ld_args.append('/MANIFESTFILE:' + temp_manifest) + + def manifest_get_embed_info(self, target_desc, ld_args): + # If a manifest should be embedded, return a tuple of + # (manifest_filename, resource_id). Returns None if no manifest + # should be embedded. See http://bugs.python.org/issue7833 for why + # we want to avoid any manifest for extension modules if we can. + for arg in ld_args: + if arg.startswith("/MANIFESTFILE:"): + temp_manifest = arg.split(":", 1)[1] + break + else: + # no /MANIFESTFILE so nothing to do. + return None + if target_desc == CCompiler.EXECUTABLE: + # by default, executables always get the manifest with the + # CRT referenced. + mfid = 1 + else: + # Extension modules try and avoid any manifest if possible. + mfid = 2 + temp_manifest = self._remove_visual_c_ref(temp_manifest) + if temp_manifest is None: + return None + return temp_manifest, mfid + + def _remove_visual_c_ref(self, manifest_file): + try: + # Remove references to the Visual C runtime, so they will + # fall through to the Visual C dependency of Python.exe. + # This way, when installed for a restricted user (e.g. + # runtimes are not in WinSxS folder, but in Python's own + # folder), the runtimes do not need to be in every folder + # with .pyd's. + # Returns either the filename of the modified manifest or + # None if no manifest should be embedded. + manifest_f = open(manifest_file) + try: + manifest_buf = manifest_f.read() + finally: + manifest_f.close() + pattern = re.compile( + r"""|)""", + re.DOTALL) + manifest_buf = re.sub(pattern, "", manifest_buf) + pattern = r"\s*" + manifest_buf = re.sub(pattern, "", manifest_buf) + # Now see if any other assemblies are referenced - if not, we + # don't want a manifest embedded. + pattern = re.compile( + r"""|)""", re.DOTALL) + if re.search(pattern, manifest_buf) is None: + return None + + manifest_f = open(manifest_file, 'w') + try: + manifest_f.write(manifest_buf) + return manifest_file + finally: + manifest_f.close() + except OSError: + pass + + # -- Miscellaneous methods ----------------------------------------- + + # Helper methods for using the MSVC registry settings + + def find_exe(self, exe): + """Return path to an MSVC executable program. + + Tries to find the program in several places: first, one of the + MSVC program search paths from the registry; next, the directories + in the PATH environment variable. If any of those work, return an + absolute path that is known to exist. If none of them work, just + return the original program name, 'exe'. + """ + for p in self.__paths: + fn = os.path.join(os.path.abspath(p), exe) + if os.path.isfile(fn): + return fn + + # didn't find it; try existing path + for p in os.environ['Path'].split(';'): + fn = os.path.join(os.path.abspath(p),exe) + if os.path.isfile(fn): + return fn + + return exe diff --git a/Tools/c-analyzer/distutils/msvccompiler.py b/Tools/c-analyzer/distutils/msvccompiler.py new file mode 100644 index 000000000000000..c0864b1f95d4e22 --- /dev/null +++ b/Tools/c-analyzer/distutils/msvccompiler.py @@ -0,0 +1,327 @@ +"""distutils.msvccompiler + +Contains MSVCCompiler, an implementation of the abstract CCompiler class +for the Microsoft Visual Studio. +""" + +# Written by Perry Stoll +# hacked by Robin Becker and Thomas Heller to do a better job of +# finding DevStudio (through the registry) + +import sys, os +from distutils.errors import DistutilsPlatformError +from distutils.ccompiler import CCompiler +from distutils import log + +_can_read_reg = False +try: + import winreg + + _can_read_reg = True + hkey_mod = winreg + + RegOpenKeyEx = winreg.OpenKeyEx + RegEnumKey = winreg.EnumKey + RegEnumValue = winreg.EnumValue + RegError = winreg.error + +except ImportError: + try: + import win32api + import win32con + _can_read_reg = True + hkey_mod = win32con + + RegOpenKeyEx = win32api.RegOpenKeyEx + RegEnumKey = win32api.RegEnumKey + RegEnumValue = win32api.RegEnumValue + RegError = win32api.error + except ImportError: + log.info("Warning: Can't read registry to find the " + "necessary compiler setting\n" + "Make sure that Python modules winreg, " + "win32api or win32con are installed.") + +if _can_read_reg: + HKEYS = (hkey_mod.HKEY_USERS, + hkey_mod.HKEY_CURRENT_USER, + hkey_mod.HKEY_LOCAL_MACHINE, + hkey_mod.HKEY_CLASSES_ROOT) + +def read_keys(base, key): + """Return list of registry keys.""" + try: + handle = RegOpenKeyEx(base, key) + except RegError: + return None + L = [] + i = 0 + while True: + try: + k = RegEnumKey(handle, i) + except RegError: + break + L.append(k) + i += 1 + return L + +def read_values(base, key): + """Return dict of registry keys and values. + + All names are converted to lowercase. + """ + try: + handle = RegOpenKeyEx(base, key) + except RegError: + return None + d = {} + i = 0 + while True: + try: + name, value, type = RegEnumValue(handle, i) + except RegError: + break + name = name.lower() + d[convert_mbcs(name)] = convert_mbcs(value) + i += 1 + return d + +def convert_mbcs(s): + dec = getattr(s, "decode", None) + if dec is not None: + try: + s = dec("mbcs") + except UnicodeError: + pass + return s + +class MacroExpander: + def __init__(self, version): + self.macros = {} + self.load_macros(version) + + def set_macro(self, macro, path, key): + for base in HKEYS: + d = read_values(base, path) + if d: + self.macros["$(%s)" % macro] = d[key] + break + + def load_macros(self, version): + vsbase = r"Software\Microsoft\VisualStudio\%0.1f" % version + self.set_macro("VCInstallDir", vsbase + r"\Setup\VC", "productdir") + self.set_macro("VSInstallDir", vsbase + r"\Setup\VS", "productdir") + net = r"Software\Microsoft\.NETFramework" + self.set_macro("FrameworkDir", net, "installroot") + try: + if version > 7.0: + self.set_macro("FrameworkSDKDir", net, "sdkinstallrootv1.1") + else: + self.set_macro("FrameworkSDKDir", net, "sdkinstallroot") + except KeyError as exc: # + raise DistutilsPlatformError( + """Python was built with Visual Studio 2003; +extensions must be built with a compiler than can generate compatible binaries. +Visual Studio 2003 was not found on this system. If you have Cygwin installed, +you can try compiling with MingW32, by passing "-c mingw32" to setup.py.""") + + p = r"Software\Microsoft\NET Framework Setup\Product" + for base in HKEYS: + try: + h = RegOpenKeyEx(base, p) + except RegError: + continue + key = RegEnumKey(h, 0) + d = read_values(base, r"%s\%s" % (p, key)) + self.macros["$(FrameworkVersion)"] = d["version"] + + def sub(self, s): + for k, v in self.macros.items(): + s = s.replace(k, v) + return s + +def get_build_version(): + """Return the version of MSVC that was used to build Python. + + For Python 2.3 and up, the version number is included in + sys.version. For earlier versions, assume the compiler is MSVC 6. + """ + prefix = "MSC v." + i = sys.version.find(prefix) + if i == -1: + return 6 + i = i + len(prefix) + s, rest = sys.version[i:].split(" ", 1) + majorVersion = int(s[:-2]) - 6 + if majorVersion >= 13: + # v13 was skipped and should be v14 + majorVersion += 1 + minorVersion = int(s[2:3]) / 10.0 + # I don't think paths are affected by minor version in version 6 + if majorVersion == 6: + minorVersion = 0 + if majorVersion >= 6: + return majorVersion + minorVersion + # else we don't know what version of the compiler this is + return None + +def get_build_architecture(): + """Return the processor architecture. + + Possible results are "Intel" or "AMD64". + """ + + prefix = " bit (" + i = sys.version.find(prefix) + if i == -1: + return "Intel" + j = sys.version.find(")", i) + return sys.version[i+len(prefix):j] + +def normalize_and_reduce_paths(paths): + """Return a list of normalized paths with duplicates removed. + + The current order of paths is maintained. + """ + # Paths are normalized so things like: /a and /a/ aren't both preserved. + reduced_paths = [] + for p in paths: + np = os.path.normpath(p) + # XXX(nnorwitz): O(n**2), if reduced_paths gets long perhaps use a set. + if np not in reduced_paths: + reduced_paths.append(np) + return reduced_paths + + +class MSVCCompiler(CCompiler) : + """Concrete class that implements an interface to Microsoft Visual C++, + as defined by the CCompiler abstract class.""" + + compiler_type = 'msvc' + + # Just set this so CCompiler's constructor doesn't barf. We currently + # don't use the 'set_executables()' bureaucracy provided by CCompiler, + # as it really isn't necessary for this sort of single-compiler class. + # Would be nice to have a consistent interface with UnixCCompiler, + # though, so it's worth thinking about. + executables = {} + + # Private class data (need to distinguish C from C++ source for compiler) + _c_extensions = ['.c'] + _cpp_extensions = ['.cc', '.cpp', '.cxx'] + _rc_extensions = ['.rc'] + _mc_extensions = ['.mc'] + + # Needed for the filename generation methods provided by the + # base class, CCompiler. + src_extensions = (_c_extensions + _cpp_extensions + + _rc_extensions + _mc_extensions) + res_extension = '.res' + obj_extension = '.obj' + static_lib_extension = '.lib' + shared_lib_extension = '.dll' + static_lib_format = shared_lib_format = '%s%s' + exe_extension = '.exe' + + def __init__(self, verbose=0, dry_run=0, force=0): + CCompiler.__init__ (self, verbose, dry_run, force) + self.__version = get_build_version() + self.__arch = get_build_architecture() + if self.__arch == "Intel": + # x86 + if self.__version >= 7: + self.__root = r"Software\Microsoft\VisualStudio" + self.__macros = MacroExpander(self.__version) + else: + self.__root = r"Software\Microsoft\Devstudio" + self.__product = "Visual Studio version %s" % self.__version + else: + # Win64. Assume this was built with the platform SDK + self.__product = "Microsoft SDK compiler %s" % (self.__version + 6) + + self.initialized = False + + + # -- Miscellaneous methods ----------------------------------------- + + # Helper methods for using the MSVC registry settings + + def find_exe(self, exe): + """Return path to an MSVC executable program. + + Tries to find the program in several places: first, one of the + MSVC program search paths from the registry; next, the directories + in the PATH environment variable. If any of those work, return an + absolute path that is known to exist. If none of them work, just + return the original program name, 'exe'. + """ + for p in self.__paths: + fn = os.path.join(os.path.abspath(p), exe) + if os.path.isfile(fn): + return fn + + # didn't find it; try existing path + for p in os.environ['Path'].split(';'): + fn = os.path.join(os.path.abspath(p),exe) + if os.path.isfile(fn): + return fn + + return exe + + def get_msvc_paths(self, path, platform='x86'): + """Get a list of devstudio directories (include, lib or path). + + Return a list of strings. The list will be empty if unable to + access the registry or appropriate registry keys not found. + """ + if not _can_read_reg: + return [] + + path = path + " dirs" + if self.__version >= 7: + key = (r"%s\%0.1f\VC\VC_OBJECTS_PLATFORM_INFO\Win32\Directories" + % (self.__root, self.__version)) + else: + key = (r"%s\6.0\Build System\Components\Platforms" + r"\Win32 (%s)\Directories" % (self.__root, platform)) + + for base in HKEYS: + d = read_values(base, key) + if d: + if self.__version >= 7: + return self.__macros.sub(d[path]).split(";") + else: + return d[path].split(";") + # MSVC 6 seems to create the registry entries we need only when + # the GUI is run. + if self.__version == 6: + for base in HKEYS: + if read_values(base, r"%s\6.0" % self.__root) is not None: + self.warn("It seems you have Visual Studio 6 installed, " + "but the expected registry settings are not present.\n" + "You must at least run the Visual Studio GUI once " + "so that these entries are created.") + break + return [] + + def set_path_env_var(self, name): + """Set environment variable 'name' to an MSVC path type value. + + This is equivalent to a SET command prior to execution of spawned + commands. + """ + + if name == "lib": + p = self.get_msvc_paths("library") + else: + p = self.get_msvc_paths(name) + if p: + os.environ[name] = ';'.join(p) + + +if get_build_version() >= 8.0: + log.debug("Importing new compiler from distutils.msvc9compiler") + OldMSVCCompiler = MSVCCompiler + from distutils.msvc9compiler import MSVCCompiler + # get_build_architecture not really relevant now we support cross-compile + from distutils.msvc9compiler import MacroExpander diff --git a/Tools/c-analyzer/distutils/spawn.py b/Tools/c-analyzer/distutils/spawn.py new file mode 100644 index 000000000000000..eb9794964d7acb9 --- /dev/null +++ b/Tools/c-analyzer/distutils/spawn.py @@ -0,0 +1,48 @@ +"""distutils.spawn + +Provides the 'spawn()' function, a front-end to various platform- +specific functions for launching another program in a sub-process. +Also provides the 'find_executable()' to search the path for a given +executable name. +""" + +import sys +import os +import os.path + + +def find_executable(executable, path=None): + """Tries to find 'executable' in the directories listed in 'path'. + + A string listing directories separated by 'os.pathsep'; defaults to + os.environ['PATH']. Returns the complete filename or None if not found. + """ + _, ext = os.path.splitext(executable) + if (sys.platform == 'win32') and (ext != '.exe'): + executable = executable + '.exe' + + if os.path.isfile(executable): + return executable + + if path is None: + path = os.environ.get('PATH', None) + if path is None: + try: + path = os.confstr("CS_PATH") + except (AttributeError, ValueError): + # os.confstr() or CS_PATH is not available + path = os.defpath + # bpo-35755: Don't use os.defpath if the PATH environment variable is + # set to an empty string + + # PATH='' doesn't match, whereas PATH=':' looks in the current directory + if not path: + return None + + paths = path.split(os.pathsep) + for p in paths: + f = os.path.join(p, executable) + if os.path.isfile(f): + # the file exists, we have a shot at spawn working + return f + return None diff --git a/Tools/c-analyzer/distutils/unixccompiler.py b/Tools/c-analyzer/distutils/unixccompiler.py new file mode 100644 index 000000000000000..1a3a8e5255070d2 --- /dev/null +++ b/Tools/c-analyzer/distutils/unixccompiler.py @@ -0,0 +1,102 @@ +"""distutils.unixccompiler + +Contains the UnixCCompiler class, a subclass of CCompiler that handles +the "typical" Unix-style command-line C compiler: + * macros defined with -Dname[=value] + * macros undefined with -Uname + * include search directories specified with -Idir + * libraries specified with -lllib + * library search directories specified with -Ldir + * compile handled by 'cc' (or similar) executable with -c option: + compiles .c to .o + * link static library handled by 'ar' command (possibly with 'ranlib') + * link shared library handled by 'cc -shared' +""" + +import os, sys, re + +from distutils.dep_util import newer +from distutils.ccompiler import CCompiler, gen_preprocess_options +from distutils.errors import DistutilsExecError, CompileError + +# XXX Things not currently handled: +# * optimization/debug/warning flags; we just use whatever's in Python's +# Makefile and live with it. Is this adequate? If not, we might +# have to have a bunch of subclasses GNUCCompiler, SGICCompiler, +# SunCCompiler, and I suspect down that road lies madness. +# * even if we don't know a warning flag from an optimization flag, +# we need some way for outsiders to feed preprocessor/compiler/linker +# flags in to us -- eg. a sysadmin might want to mandate certain flags +# via a site config file, or a user might want to set something for +# compiling this module distribution only via the setup.py command +# line, whatever. As long as these options come from something on the +# current system, they can be as system-dependent as they like, and we +# should just happily stuff them into the preprocessor/compiler/linker +# options and carry on. + + +class UnixCCompiler(CCompiler): + + compiler_type = 'unix' + + # These are used by CCompiler in two places: the constructor sets + # instance attributes 'preprocessor', 'compiler', etc. from them, and + # 'set_executable()' allows any of these to be set. The defaults here + # are pretty generic; they will probably have to be set by an outsider + # (eg. using information discovered by the sysconfig about building + # Python extensions). + executables = {'preprocessor' : None, + 'compiler' : ["cc"], + 'compiler_so' : ["cc"], + 'compiler_cxx' : ["cc"], + 'linker_so' : ["cc", "-shared"], + 'linker_exe' : ["cc"], + 'archiver' : ["ar", "-cr"], + 'ranlib' : None, + } + + if sys.platform[:6] == "darwin": + executables['ranlib'] = ["ranlib"] + + # Needed for the filename generation methods provided by the base + # class, CCompiler. NB. whoever instantiates/uses a particular + # UnixCCompiler instance should set 'shared_lib_ext' -- we set a + # reasonable common default here, but it's not necessarily used on all + # Unices! + + src_extensions = [".c",".C",".cc",".cxx",".cpp",".m"] + obj_extension = ".o" + static_lib_extension = ".a" + shared_lib_extension = ".so" + dylib_lib_extension = ".dylib" + xcode_stub_lib_extension = ".tbd" + static_lib_format = shared_lib_format = dylib_lib_format = "lib%s%s" + xcode_stub_lib_format = dylib_lib_format + if sys.platform == "cygwin": + exe_extension = ".exe" + + def preprocess(self, source, output_file=None, macros=None, + include_dirs=None, extra_preargs=None, extra_postargs=None): + fixed_args = self._fix_compile_args(None, macros, include_dirs) + ignore, macros, include_dirs = fixed_args + pp_opts = gen_preprocess_options(macros, include_dirs) + pp_args = self.preprocessor + pp_opts + if output_file: + pp_args.extend(['-o', output_file]) + if extra_preargs: + pp_args[:0] = extra_preargs + if extra_postargs: + pp_args.extend(extra_postargs) + pp_args.append(source) + + # We need to preprocess: either we're being forced to, or we're + # generating output to stdout, or there's a target output file and + # the source file is newer than the target (or the target doesn't + # exist). + if self.force or output_file is None or newer(source, output_file): + if output_file: + self.mkpath(os.path.dirname(output_file)) + try: + self.spawn(pp_args) + except DistutilsExecError as msg: + raise CompileError(msg) diff --git a/Tools/c-analyzer/distutils/util.py b/Tools/c-analyzer/distutils/util.py new file mode 100644 index 000000000000000..89ca094336fdb8f --- /dev/null +++ b/Tools/c-analyzer/distutils/util.py @@ -0,0 +1,171 @@ +"""distutils.util + +Miscellaneous utility functions -- anything that doesn't fit into +one of the other *util.py modules. +""" + +import os +import re +import string +import sys +from distutils.errors import DistutilsPlatformError + +def get_host_platform(): + """Return a string that identifies the current platform. This is used mainly to + distinguish platform-specific build directories and platform-specific built + distributions. Typically includes the OS name and version and the + architecture (as supplied by 'os.uname()'), although the exact information + included depends on the OS; eg. on Linux, the kernel version isn't + particularly important. + + Examples of returned values: + linux-i586 + linux-alpha (?) + solaris-2.6-sun4u + + Windows will return one of: + win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc) + win32 (all others - specifically, sys.platform is returned) + + For other non-POSIX platforms, currently just returns 'sys.platform'. + + """ + if os.name == 'nt': + if 'amd64' in sys.version.lower(): + return 'win-amd64' + if '(arm)' in sys.version.lower(): + return 'win-arm32' + if '(arm64)' in sys.version.lower(): + return 'win-arm64' + return sys.platform + + # Set for cross builds explicitly + if "_PYTHON_HOST_PLATFORM" in os.environ: + return os.environ["_PYTHON_HOST_PLATFORM"] + + if os.name != "posix" or not hasattr(os, 'uname'): + # XXX what about the architecture? NT is Intel or Alpha, + # Mac OS is M68k or PPC, etc. + return sys.platform + + # Try to distinguish various flavours of Unix + + (osname, host, release, version, machine) = os.uname() + + # Convert the OS name to lowercase, remove '/' characters, and translate + # spaces (for "Power Macintosh") + osname = osname.lower().replace('/', '') + machine = machine.replace(' ', '_') + machine = machine.replace('/', '-') + + if osname[:5] == "linux": + # At least on Linux/Intel, 'machine' is the processor -- + # i386, etc. + # XXX what about Alpha, SPARC, etc? + return "%s-%s" % (osname, machine) + elif osname[:5] == "sunos": + if release[0] >= "5": # SunOS 5 == Solaris 2 + osname = "solaris" + release = "%d.%s" % (int(release[0]) - 3, release[2:]) + # We can't use "platform.architecture()[0]" because a + # bootstrap problem. We use a dict to get an error + # if some suspicious happens. + bitness = {2147483647:"32bit", 9223372036854775807:"64bit"} + machine += ".%s" % bitness[sys.maxsize] + # fall through to standard osname-release-machine representation + elif osname[:3] == "aix": + from _aix_support import aix_platform + return aix_platform() + elif osname[:6] == "cygwin": + osname = "cygwin" + rel_re = re.compile (r'[\d.]+', re.ASCII) + m = rel_re.match(release) + if m: + release = m.group() + elif osname[:6] == "darwin": + import _osx_support, sysconfig + osname, release, machine = _osx_support.get_platform_osx( + sysconfig.get_config_vars(), + osname, release, machine) + + return "%s-%s-%s" % (osname, release, machine) + +def get_platform(): + if os.name == 'nt': + TARGET_TO_PLAT = { + 'x86' : 'win32', + 'x64' : 'win-amd64', + 'arm' : 'win-arm32', + } + return TARGET_TO_PLAT.get(os.environ.get('VSCMD_ARG_TGT_ARCH')) or get_host_platform() + else: + return get_host_platform() + + +# Needed by 'split_quoted()' +_wordchars_re = _squote_re = _dquote_re = None +def _init_regex(): + global _wordchars_re, _squote_re, _dquote_re + _wordchars_re = re.compile(r'[^\\\'\"%s ]*' % string.whitespace) + _squote_re = re.compile(r"'(?:[^'\\]|\\.)*'") + _dquote_re = re.compile(r'"(?:[^"\\]|\\.)*"') + +def split_quoted (s): + """Split a string up according to Unix shell-like rules for quotes and + backslashes. In short: words are delimited by spaces, as long as those + spaces are not escaped by a backslash, or inside a quoted string. + Single and double quotes are equivalent, and the quote characters can + be backslash-escaped. The backslash is stripped from any two-character + escape sequence, leaving only the escaped character. The quote + characters are stripped from any quoted string. Returns a list of + words. + """ + + # This is a nice algorithm for splitting up a single string, since it + # doesn't require character-by-character examination. It was a little + # bit of a brain-bender to get it working right, though... + if _wordchars_re is None: _init_regex() + + s = s.strip() + words = [] + pos = 0 + + while s: + m = _wordchars_re.match(s, pos) + end = m.end() + if end == len(s): + words.append(s[:end]) + break + + if s[end] in string.whitespace: # unescaped, unquoted whitespace: now + words.append(s[:end]) # we definitely have a word delimiter + s = s[end:].lstrip() + pos = 0 + + elif s[end] == '\\': # preserve whatever is being escaped; + # will become part of the current word + s = s[:end] + s[end+1:] + pos = end+1 + + else: + if s[end] == "'": # slurp singly-quoted string + m = _squote_re.match(s, end) + elif s[end] == '"': # slurp doubly-quoted string + m = _dquote_re.match(s, end) + else: + raise RuntimeError("this can't happen (bad char '%c')" % s[end]) + + if m is None: + raise ValueError("bad string (mismatched %s quotes?)" % s[end]) + + (beg, end) = m.span() + s = s[:beg] + s[beg+1:end-1] + s[end:] + pos = m.end() - 2 + + if pos >= len(s): + words.append(s) + break + + return words + +# split_quoted () From c6858d1e7f4cd3184d5ddea4025ad5dfc7596546 Mon Sep 17 00:00:00 2001 From: Max Bachmann Date: Thu, 9 Mar 2023 22:09:12 +0100 Subject: [PATCH 49/97] gh-102255: Improve build support for Windows API partitions (GH-102256) Add `MS_WINDOWS_DESKTOP`, `MS_WINDOWS_APPS`, `MS_WINDOWS_SYSTEM` and `MS_WINDOWS_GAMES` preprocessor definitions to allow switching off functionality missing from particular API partitions ("partitions" are used in Windows to identify overlapping subsets of APIs). CPython only officially supports `MS_WINDOWS_DESKTOP` and `MS_WINDOWS_SYSTEM` (APPS is included by normal desktop builds, but APPS without DESKTOP is not covered). Other configurations are a convenience for people building their own runtimes. `MS_WINDOWS_GAMES` is for the Xbox subset of the Windows API, which is also available on client OS, but is restricted compared to `MS_WINDOWS_DESKTOP`. These restrictions may change over time, as they relate to the build headers rather than the OS support, and so we assume that Xbox builds will use the latest available version of the GDK. --- Include/internal/pycore_fileutils.h | 8 + Lib/test/test_os.py | 8 +- ...-02-26-11-43-56.gh-issue-102255.cRnI5x.rst | 1 + Modules/_io/_iomodule.c | 6 +- Modules/_io/_iomodule.h | 6 +- Modules/_io/clinic/winconsoleio.c.h | 42 ++-- Modules/_io/fileio.c | 2 + Modules/_io/winconsoleio.c | 6 +- Modules/_localemodule.c | 8 +- Modules/_multiprocessing/multiprocessing.h | 4 +- Modules/_pickle.c | 6 + Modules/_randommodule.c | 6 +- Modules/_ssl.c | 4 + Modules/_winapi.c | 13 ++ Modules/clinic/posixmodule.c.h | 10 +- Modules/errnomodule.c | 2 + Modules/faulthandler.c | 2 +- Modules/getpath.c | 2 + Modules/mmapmodule.c | 8 +- Modules/posixmodule.c | 58 +++-- Modules/selectmodule.c | 6 +- Modules/socketmodule.c | 60 +++-- Modules/timemodule.c | 8 +- Objects/obmalloc.c | 1 - PC/clinic/msvcrtmodule.c.h | 42 +++- PC/clinic/winreg.c.h | 218 +++++++++++++++++- PC/config.c | 8 + PC/config_minimal.c | 6 + PC/msvcrtmodule.c | 22 ++ PC/pyconfig.h | 24 +- PC/winreg.c | 25 ++ Parser/myreadline.c | 8 +- Python/dynload_win.c | 8 + Python/fileutils.c | 89 ++++++- Python/pylifecycle.c | 2 +- Python/sysmodule.c | 4 + 36 files changed, 633 insertions(+), 100 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-26-11-43-56.gh-issue-102255.cRnI5x.rst diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index f8e2bf225908887..445ac0a3d955d93 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -251,6 +251,14 @@ extern int _Py_add_relfile(wchar_t *dirname, extern size_t _Py_find_basename(const wchar_t *filename); PyAPI_FUNC(wchar_t *) _Py_normpath(wchar_t *path, Py_ssize_t size); +// The Windows Games API family does not provide these functions +// so provide our own implementations. Remove them in case they get added +// to the Games API family +#if defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) +#include + +extern HRESULT PathCchSkipRoot(const wchar_t *pszPath, const wchar_t **ppszRootEnd); +#endif /* defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) */ // Macros to protect CRT calls against instant termination when passed an // invalid parameter (bpo-23524). IPH stands for Invalid Parameter Handler. diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 792794ca1094892..ba6feb69ea17125 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -3084,11 +3084,13 @@ def test_device_encoding(self): class PidTests(unittest.TestCase): @unittest.skipUnless(hasattr(os, 'getppid'), "test needs os.getppid") def test_getppid(self): - p = subprocess.Popen([sys.executable, '-c', + p = subprocess.Popen([sys._base_executable, '-c', 'import os; print(os.getppid())'], - stdout=subprocess.PIPE) - stdout, _ = p.communicate() + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, error = p.communicate() # We are the parent of our subprocess + self.assertEqual(error, b'') self.assertEqual(int(stdout), os.getpid()) def check_waitpid(self, code, exitcode, callback=None): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-26-11-43-56.gh-issue-102255.cRnI5x.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-26-11-43-56.gh-issue-102255.cRnI5x.rst new file mode 100644 index 000000000000000..daabc3c15f6ee2a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-26-11-43-56.gh-issue-102255.cRnI5x.rst @@ -0,0 +1 @@ +Improve build support for the Xbox. Patch by Max Bachmann. diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index d8d836b8382eb18..1506755427fc0d6 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -317,7 +317,7 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode, _PyIO_State *state = get_io_state(module); { PyObject *RawIO_class = (PyObject *)state->PyFileIO_Type; -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO const PyConfig *config = _Py_GetConfig(); if (!config->legacy_windows_stdio && _PyIO_get_console_type(path_or_fd) != '\0') { RawIO_class = (PyObject *)&PyWindowsConsoleIO_Type; @@ -660,7 +660,7 @@ static PyTypeObject* static_types[] = { // PyRawIOBase_Type(PyIOBase_Type) subclasses &_PyBytesIOBuffer_Type, -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO &PyWindowsConsoleIO_Type, #endif }; @@ -718,7 +718,7 @@ PyInit__io(void) } // Set type base classes -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO PyWindowsConsoleIO_Type.tp_base = &PyRawIOBase_Type; #endif diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h index 02daef9e85677e8..d7224e56f9a7224 100644 --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -26,9 +26,9 @@ extern PyType_Spec fileio_spec; extern PyType_Spec stringio_spec; extern PyType_Spec textiowrapper_spec; -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO extern PyTypeObject PyWindowsConsoleIO_Type; -#endif /* MS_WINDOWS */ +#endif /* HAVE_WINDOWS_CONSOLE_IO */ /* These functions are used as METH_NOARGS methods, are normally called * with args=NULL, and return a new reference. @@ -178,7 +178,7 @@ find_io_state_by_def(PyTypeObject *type) extern _PyIO_State *_PyIO_get_module_state(void); -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO extern char _PyIO_get_console_type(PyObject *); #endif diff --git a/Modules/_io/clinic/winconsoleio.c.h b/Modules/_io/clinic/winconsoleio.c.h index df834dbde40f5b2..4c5cd0892c4a6d2 100644 --- a/Modules/_io/clinic/winconsoleio.c.h +++ b/Modules/_io/clinic/winconsoleio.c.h @@ -8,7 +8,7 @@ preserve #endif -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_close__doc__, "close($self, /)\n" @@ -31,9 +31,9 @@ _io__WindowsConsoleIO_close(winconsoleio *self, PyObject *Py_UNUSED(ignored)) return _io__WindowsConsoleIO_close_impl(self); } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO___init____doc__, "_WindowsConsoleIO(file, mode=\'r\', closefd=True, opener=None)\n" @@ -131,9 +131,9 @@ _io__WindowsConsoleIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) return return_value; } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_fileno__doc__, "fileno($self, /)\n" @@ -153,9 +153,9 @@ _io__WindowsConsoleIO_fileno(winconsoleio *self, PyObject *Py_UNUSED(ignored)) return _io__WindowsConsoleIO_fileno_impl(self); } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_readable__doc__, "readable($self, /)\n" @@ -175,9 +175,9 @@ _io__WindowsConsoleIO_readable(winconsoleio *self, PyObject *Py_UNUSED(ignored)) return _io__WindowsConsoleIO_readable_impl(self); } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_writable__doc__, "writable($self, /)\n" @@ -197,9 +197,9 @@ _io__WindowsConsoleIO_writable(winconsoleio *self, PyObject *Py_UNUSED(ignored)) return _io__WindowsConsoleIO_writable_impl(self); } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_readinto__doc__, "readinto($self, buffer, /)\n" @@ -239,9 +239,9 @@ _io__WindowsConsoleIO_readinto(winconsoleio *self, PyObject *arg) return return_value; } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_readall__doc__, "readall($self, /)\n" @@ -263,9 +263,9 @@ _io__WindowsConsoleIO_readall(winconsoleio *self, PyObject *Py_UNUSED(ignored)) return _io__WindowsConsoleIO_readall_impl(self); } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_read__doc__, "read($self, size=-1, /)\n" @@ -305,9 +305,9 @@ _io__WindowsConsoleIO_read(winconsoleio *self, PyObject *const *args, Py_ssize_t return return_value; } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_write__doc__, "write($self, b, /)\n" @@ -348,9 +348,9 @@ _io__WindowsConsoleIO_write(winconsoleio *self, PyObject *arg) return return_value; } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_isatty__doc__, "isatty($self, /)\n" @@ -370,7 +370,7 @@ _io__WindowsConsoleIO_isatty(winconsoleio *self, PyObject *Py_UNUSED(ignored)) return _io__WindowsConsoleIO_isatty_impl(self); } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ #ifndef _IO__WINDOWSCONSOLEIO_CLOSE_METHODDEF #define _IO__WINDOWSCONSOLEIO_CLOSE_METHODDEF @@ -407,4 +407,4 @@ _io__WindowsConsoleIO_isatty(winconsoleio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #define _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #endif /* !defined(_IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF) */ -/*[clinic end generated code: output=4920e9068e0cf08a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=163e934aa9b0ef16 input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 35a498ce5a83544..1118d86e6c9a10f 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -37,7 +37,9 @@ #ifdef MS_WINDOWS /* can simulate truncate with Win32 API functions; see file_truncate */ #define HAVE_FTRUNCATE +#ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN +#endif #include #endif diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index de07b50f5ce4cbd..f836e230243020c 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -11,7 +11,7 @@ #include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH #include "pycore_object.h" // _PyObject_GC_UNTRACK() -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO #include "structmember.h" // PyMemberDef #ifdef HAVE_SYS_TYPES_H @@ -22,7 +22,9 @@ #endif #include /* For offsetof */ +#ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN +#endif #include #include @@ -1177,4 +1179,4 @@ PyTypeObject PyWindowsConsoleIO_Type = { 0, /* tp_finalize */ }; -#endif /* MS_WINDOWS */ +#endif /* HAVE_WINDOWS_CONSOLE_IO */ diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index 23c38e14d997d1b..96675cdfb661ad7 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -35,7 +35,9 @@ This software comes with no warranty. Use at your own risk. #endif #if defined(MS_WINDOWS) +#ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN +#endif #include #endif @@ -457,12 +459,12 @@ _locale__getdefaultlocale_impl(PyObject *module) PyOS_snprintf(encoding, sizeof(encoding), "cp%u", GetACP()); - if (GetLocaleInfo(LOCALE_USER_DEFAULT, + if (GetLocaleInfoA(LOCALE_USER_DEFAULT, LOCALE_SISO639LANGNAME, locale, sizeof(locale))) { Py_ssize_t i = strlen(locale); locale[i++] = '_'; - if (GetLocaleInfo(LOCALE_USER_DEFAULT, + if (GetLocaleInfoA(LOCALE_USER_DEFAULT, LOCALE_SISO3166CTRYNAME, locale+i, (int)(sizeof(locale)-i))) return Py_BuildValue("ss", locale, encoding); @@ -474,7 +476,7 @@ _locale__getdefaultlocale_impl(PyObject *module) locale[0] = '0'; locale[1] = 'x'; - if (GetLocaleInfo(LOCALE_USER_DEFAULT, LOCALE_IDEFAULTLANGUAGE, + if (GetLocaleInfoA(LOCALE_USER_DEFAULT, LOCALE_IDEFAULTLANGUAGE, locale+2, sizeof(locale)-2)) { return Py_BuildValue("ss", locale, encoding); } diff --git a/Modules/_multiprocessing/multiprocessing.h b/Modules/_multiprocessing/multiprocessing.h index b595e5a8dd18de4..dfc2a8e0799a603 100644 --- a/Modules/_multiprocessing/multiprocessing.h +++ b/Modules/_multiprocessing/multiprocessing.h @@ -12,7 +12,9 @@ */ #ifdef MS_WINDOWS -# define WIN32_LEAN_AND_MEAN +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif # include # include # include /* getpid() */ diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 1b34977806b661c..a26732af8ba2a1f 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -42,6 +42,12 @@ enum { #define FLOAT FLOAT_ #define INT INT_ #define LONG LONG_ + +/* This can already be defined on Windows to set the character set + the Windows header files treat as default */ +#ifdef UNICODE +#undef UNICODE +#endif #endif /* Pickle opcodes. These must be kept updated with pickle.py. diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 68060c07033d345..6e22053239305a5 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -77,6 +77,10 @@ # include // getpid() #endif +#ifdef MS_WINDOWS +# include +#endif + /* Period parameters -- These are all magic. Don't change. */ #define N 624 #define M 397 @@ -259,7 +263,7 @@ random_seed_time_pid(RandomObject *self) key[0] = (uint32_t)(now & 0xffffffffU); key[1] = (uint32_t)(now >> 32); -#ifdef MS_WINDOWS_NON_DESKTOP +#if defined(MS_WINDOWS) && !defined(MS_WINDOWS_DESKTOP) && !defined(MS_WINDOWS_SYSTEM) key[2] = (uint32_t)GetCurrentProcessId(); #elif defined(HAVE_GETPID) key[2] = (uint32_t)getpid(); diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 8f03a846aed0891..28112317bc289ea 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -28,6 +28,10 @@ /* Include symbols from _socket module */ #include "socketmodule.h" +#ifdef MS_WINDOWS +# include +#endif + #include "_ssl.h" /* Redefined below for Windows debug builds after important #includes */ diff --git a/Modules/_winapi.c b/Modules/_winapi.c index eefc2571ee8287d..83cde7501176b64 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -39,8 +39,11 @@ #include "structmember.h" // PyMemberDef +#ifndef WINDOWS_LEAN_AND_MEAN #define WINDOWS_LEAN_AND_MEAN +#endif #include "windows.h" +#include #include #include "winreparse.h" @@ -63,6 +66,14 @@ #define T_HANDLE T_POINTER +// winbase.h limits the STARTF_* flags to the desktop API as of 10.0.19041. +#ifndef STARTF_USESHOWWINDOW +#define STARTF_USESHOWWINDOW 0x00000001 +#endif +#ifndef STARTF_USESTDHANDLES +#define STARTF_USESTDHANDLES 0x00000100 +#endif + typedef struct { PyTypeObject *overlapped_type; } WinApiState; @@ -1201,8 +1212,10 @@ _winapi_ExitProcess_impl(PyObject *module, UINT ExitCode) /*[clinic end generated code: output=a387deb651175301 input=4f05466a9406c558]*/ { #if defined(Py_DEBUG) +#ifdef MS_WINDOWS_DESKTOP SetErrorMode(SEM_FAILCRITICALERRORS|SEM_NOALIGNMENTFAULTEXCEPT| SEM_NOGPFAULTERRORBOX|SEM_NOOPENFILEERRORBOX); +#endif _CrtSetReportMode(_CRT_ASSERT, _CRTDBG_MODE_DEBUG); #endif diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index dcd25c28370c930..6565f8df935cb85 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -10988,7 +10988,7 @@ os_getrandom(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #endif /* defined(HAVE_GETRANDOM_SYSCALL) */ -#if defined(MS_WINDOWS) +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM)) PyDoc_STRVAR(os__add_dll_directory__doc__, "_add_dll_directory($module, /, path)\n" @@ -11057,9 +11057,9 @@ os__add_dll_directory(PyObject *module, PyObject *const *args, Py_ssize_t nargs, return return_value; } -#endif /* defined(MS_WINDOWS) */ +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM)) */ -#if defined(MS_WINDOWS) +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM)) PyDoc_STRVAR(os__remove_dll_directory__doc__, "_remove_dll_directory($module, /, cookie)\n" @@ -11120,7 +11120,7 @@ os__remove_dll_directory(PyObject *module, PyObject *const *args, Py_ssize_t nar return return_value; } -#endif /* defined(MS_WINDOWS) */ +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM)) */ #if (defined(WIFEXITED) || defined(MS_WINDOWS)) @@ -11796,4 +11796,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=1b0eb6a76b1a0e28 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9495478e51701b8a input=a9049054013a1b77]*/ diff --git a/Modules/errnomodule.c b/Modules/errnomodule.c index 4de4144520aa48e..df4e494ba8a9737 100644 --- a/Modules/errnomodule.c +++ b/Modules/errnomodule.c @@ -5,7 +5,9 @@ /* Windows socket errors (WSA*) */ #ifdef MS_WINDOWS +#ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN +#endif #include /* The following constants were added to errno.h in VS2010 but have preferred WSA equivalents. */ diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 5309a3728c5e07b..bfe35fed7a450a3 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -953,7 +953,7 @@ faulthandler_unregister_py(PyObject *self, PyObject *args) static void faulthandler_suppress_crash_report(void) { -#ifdef MS_WINDOWS +#ifdef MS_WINDOWS_DESKTOP UINT mode; /* Configure Windows to not display the Windows Error Reporting dialog */ diff --git a/Modules/getpath.c b/Modules/getpath.c index c807a3c8e9a2b90..2f20521592ce2e2 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -745,10 +745,12 @@ static int library_to_dict(PyObject *dict, const char *key) { #ifdef MS_WINDOWS +#ifdef Py_ENABLE_SHARED extern HMODULE PyWin_DLLhModule; if (PyWin_DLLhModule) { return winmodule_to_dict(dict, key, PyWin_DLLhModule); } +#endif #elif defined(WITH_NEXT_FRAMEWORK) static char modPath[MAXPATHLEN + 1]; static int modPathInitialized = -1; diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 6054c2853d7a78d..fe76ca6eafaa88a 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -29,6 +29,10 @@ #include "structmember.h" // PyMemberDef #include // offsetof() +// to support MS_WINDOWS_SYSTEM OpenFileMappingA / CreateFileMappingA +// need to be replaced with OpenFileMappingW / CreateFileMappingW +#if !defined(MS_WINDOWS) || defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_GAMES) + #ifndef MS_WINDOWS #define UNIX # ifdef HAVE_FCNTL_H @@ -647,7 +651,7 @@ mmap_flush_method(mmap_object *self, PyObject *args) if (self->access == ACCESS_READ || self->access == ACCESS_COPY) Py_RETURN_NONE; -#ifdef MS_WINDOWS +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM) if (!FlushViewOfFile(self->data+offset, size)) { PyErr_SetFromWindowsErr(GetLastError()); return NULL; @@ -1724,3 +1728,5 @@ PyInit_mmap(void) { return PyModuleDef_Init(&mmapmodule); } + +#endif /* !MS_WINDOWS || MS_WINDOWS_DESKTOP || MS_WINDOWS_GAMES */ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index a3d86cbe7a57fb2..0d534f35f6a420a 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -26,11 +26,15 @@ #ifdef MS_WINDOWS # include -# include +# if !defined(MS_WINDOWS_GAMES) || defined(MS_WINDOWS_DESKTOP) +# include +# endif # include # include // UNLEN # include "osdefs.h" // SEP -# define HAVE_SYMLINK +# if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) +# define HAVE_SYMLINK +# endif /* MS_WINDOWS_DESKTOP | MS_WINDOWS_SYSTEM */ #endif #include "structmember.h" // PyMemberDef @@ -311,7 +315,7 @@ corresponding Unix manual entries for more information on calls."); # include #endif -#if defined(MS_WINDOWS) +#ifdef HAVE_WINDOWS_CONSOLE_IO # define TERMSIZE_USE_CONIO #elif defined(HAVE_SYS_IOCTL_H) # include @@ -321,7 +325,7 @@ corresponding Unix manual entries for more information on calls."); # if defined(TIOCGWINSZ) # define TERMSIZE_USE_IOCTL # endif -#endif /* MS_WINDOWS */ +#endif /* HAVE_WINDOWS_CONSOLE_IO */ /* Various compilers have only certain posix functions */ /* XXX Gosh I wish these were all moved into pyconfig.h */ @@ -329,21 +333,25 @@ corresponding Unix manual entries for more information on calls."); # define HAVE_OPENDIR 1 # define HAVE_SYSTEM 1 # include -#else -# ifdef _MSC_VER - /* Microsoft compiler */ +#elif defined( _MSC_VER) + /* Microsoft compiler */ +# if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM) # define HAVE_GETPPID 1 +# endif /* MS_WINDOWS_DESKTOP | MS_WINDOWS_APP | MS_WINDOWS_SYSTEM */ +# if defined(MS_WINDOWS_DESKTOP) # define HAVE_GETLOGIN 1 +# endif /* MS_WINDOWS_DESKTOP */ +# if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) # define HAVE_SPAWNV 1 # define HAVE_EXECV 1 # define HAVE_WSPAWNV 1 # define HAVE_WEXECV 1 -# define HAVE_PIPE 1 # define HAVE_SYSTEM 1 # define HAVE_CWAIT 1 -# define HAVE_FSYNC 1 -# define fsync _commit -# endif /* _MSC_VER */ +# endif /* MS_WINDOWS_DESKTOP | MS_WINDOWS_SYSTEM */ +# define HAVE_PIPE 1 +# define HAVE_FSYNC 1 +# define fsync _commit #endif /* ! __WATCOMC__ || __QNX__ */ /*[clinic input] @@ -1536,7 +1544,7 @@ convertenviron(void) #ifdef MS_WINDOWS /* _wenviron must be initialized in this way if the program is started through main() instead of wmain(). */ - _wgetenv(L""); + (void)_wgetenv(L""); e = _wenviron; #elif defined(WITH_NEXT_FRAMEWORK) || (defined(__APPLE__) && defined(Py_ENABLE_SHARED)) /* environ is not accessible as an extern in a shared object on OSX; use @@ -1785,6 +1793,10 @@ attributes_from_dir(LPCWSTR pszFile, BY_HANDLE_FILE_INFORMATION *info, ULONG *re if (n && (pszFile[n - 1] == L'\\' || pszFile[n - 1] == L'/')) { // cannot use PyMem_Malloc here because we do not hold the GIL filename = (LPCWSTR)malloc((n + 1) * sizeof(filename[0])); + if(!filename) { + SetLastError(ERROR_NOT_ENOUGH_MEMORY); + return FALSE; + } wcsncpy_s((LPWSTR)filename, n + 1, pszFile, n); while (--n > 0 && (filename[n] == L'\\' || filename[n] == L'/')) { ((LPWSTR)filename)[n] = L'\0'; @@ -7933,10 +7945,10 @@ static PyObject * os_getpid_impl(PyObject *module) /*[clinic end generated code: output=9ea6fdac01ed2b3c input=5a9a00f0ab68aa00]*/ { -#ifdef MS_WINDOWS_NON_DESKTOP - return PyLong_FromUnsignedLong(GetCurrentProcessId()); -#else +#if !defined(MS_WINDOWS) || defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) return PyLong_FromPid(getpid()); +#else + return PyLong_FromUnsignedLong(GetCurrentProcessId()); #endif } #endif /* defined(HAVE_GETPID) */ @@ -8392,6 +8404,7 @@ os_kill_impl(PyObject *module, pid_t pid, Py_ssize_t signal) DWORD err; HANDLE handle; +#ifdef HAVE_WINDOWS_CONSOLE_IO /* Console processes which share a common console can be sent CTRL+C or CTRL+BREAK events, provided they handle said events. */ if (sig == CTRL_C_EVENT || sig == CTRL_BREAK_EVENT) { @@ -8399,9 +8412,11 @@ os_kill_impl(PyObject *module, pid_t pid, Py_ssize_t signal) err = GetLastError(); PyErr_SetFromWindowsErr(err); } - else + else { Py_RETURN_NONE; + } } +#endif /* HAVE_WINDOWS_CONSOLE_IO */ /* If the signal is outside of what GenerateConsoleCtrlEvent can use, attempt to open and terminate the process. */ @@ -13776,7 +13791,9 @@ os_cpu_count_impl(PyObject *module) { int ncpu = 0; #ifdef MS_WINDOWS +#ifdef MS_WINDOWS_DESKTOP ncpu = GetActiveProcessorCount(ALL_PROCESSOR_GROUPS); +#endif #elif defined(__hpux) ncpu = mpctl(MPC_GETNUMSPUS, NULL, NULL); #elif defined(HAVE_SYSCONF) && defined(_SC_NPROCESSORS_ONLN) @@ -13848,6 +13865,10 @@ os_set_inheritable_impl(PyObject *module, int fd, int inheritable) #ifdef MS_WINDOWS +#ifndef HANDLE_FLAG_INHERIT +#define HANDLE_FLAG_INHERIT 0x00000001 +#endif + /*[clinic input] os.get_handle_inheritable -> bool handle: intptr_t @@ -15023,7 +15044,8 @@ os_getrandom_impl(PyObject *module, Py_ssize_t size, int flags) } #endif /* HAVE_GETRANDOM_SYSCALL */ -#ifdef MS_WINDOWS +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM) + /* bpo-36085: Helper functions for managing DLL search directories * on win32 */ @@ -15114,7 +15136,7 @@ os__remove_dll_directory_impl(PyObject *module, PyObject *cookie) Py_RETURN_NONE; } -#endif +#endif /* MS_WINDOWS_APP || MS_WINDOWS_SYSTEM */ /* Only check if WIFEXITED is available: expect that it comes diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index df4043de08daccc..5a1e40d0b4a482c 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -57,8 +57,10 @@ extern void bzero(void *, int); #endif #ifdef MS_WINDOWS -# define WIN32_LEAN_AND_MEAN -# include +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include #else # define SOCKET int #endif diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 43a0cc0f963f9d9..b7927750e334b77 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -270,7 +270,7 @@ shutdown(how) -- shut down traffic in one or both directions\n\ # include -#else +#else /* MS_WINDOWS */ /* MS_WINDOWS includes */ # ifdef HAVE_FCNTL_H @@ -281,7 +281,6 @@ shutdown(how) -- shut down traffic in one or both directions\n\ # include /* Macros based on the IPPROTO enum, see: https://bugs.python.org/issue29515 */ -#ifdef MS_WINDOWS #define IPPROTO_ICMP IPPROTO_ICMP #define IPPROTO_IGMP IPPROTO_IGMP #define IPPROTO_GGP IPPROTO_GGP @@ -312,7 +311,6 @@ shutdown(how) -- shut down traffic in one or both directions\n\ #define IPPROTO_PGM IPPROTO_PGM // WinSock2 only #define IPPROTO_L2TP IPPROTO_L2TP // WinSock2 only #define IPPROTO_SCTP IPPROTO_SCTP // WinSock2 only -#endif /* MS_WINDOWS */ /* Provides the IsWindows7SP1OrGreater() function */ #include @@ -348,13 +346,18 @@ remove_unusable_flags(PyObject *m) { PyObject *dict; OSVERSIONINFOEX info; - DWORDLONG dwlConditionMask; dict = PyModule_GetDict(m); if (dict == NULL) { return -1; } - +#ifndef MS_WINDOWS_DESKTOP + info.dwOSVersionInfoSize = sizeof(info); + if (!GetVersionExW((OSVERSIONINFOW*) &info)) { + PyErr_SetFromWindowsErr(0); + return -1; + } +#else /* set to Windows 10, except BuildNumber. */ memset(&info, 0, sizeof(info)); info.dwOSVersionInfoSize = sizeof(info); @@ -362,19 +365,30 @@ remove_unusable_flags(PyObject *m) info.dwMinorVersion = 0; /* set Condition Mask */ - dwlConditionMask = 0; + DWORDLONG dwlConditionMask = 0; VER_SET_CONDITION(dwlConditionMask, VER_MAJORVERSION, VER_GREATER_EQUAL); VER_SET_CONDITION(dwlConditionMask, VER_MINORVERSION, VER_GREATER_EQUAL); VER_SET_CONDITION(dwlConditionMask, VER_BUILDNUMBER, VER_GREATER_EQUAL); +#endif for (int i=0; i 10 || + (info.dwMajorVersion == 10 && info.dwMinorVersion > 0) || + (info.dwMajorVersion == 10 && info.dwMinorVersion == 0 && + info.dwBuildNumber >= win_runtime_flags[i].build_number); +#endif + if (isSupported) { break; } else { @@ -497,14 +511,14 @@ remove_unusable_flags(PyObject *m) #endif #endif -#ifdef MS_WINDOWS +#ifdef MS_WINDOWS_DESKTOP #define sockaddr_rc SOCKADDR_BTH_REDEF #define USE_BLUETOOTH 1 #define AF_BLUETOOTH AF_BTH #define BTPROTO_RFCOMM BTHPROTO_RFCOMM #define _BT_RC_MEMB(sa, memb) ((sa)->memb) -#endif +#endif /* MS_WINDOWS_DESKTOP */ /* Convert "sock_addr_t *" to "struct sockaddr *". */ #define SAS2SA(x) (&((x)->sa)) @@ -2869,11 +2883,16 @@ sock_accept(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) newfd = ctx.result; #ifdef MS_WINDOWS +#if defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) +#ifndef HANDLE_FLAG_INHERIT +#define HANDLE_FLAG_INHERIT 0x00000001 +#endif if (!SetHandleInformation((HANDLE)newfd, HANDLE_FLAG_INHERIT, 0)) { PyErr_SetFromWindowsErr(0); SOCKETCLOSE(newfd); goto finally; } +#endif #else #if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) @@ -5434,11 +5453,6 @@ sock_initobj_impl(PySocketSockObject *self, int family, int type, int proto, proto = 0; } #ifdef MS_WINDOWS - /* Windows implementation */ -#ifndef WSA_FLAG_NO_HANDLE_INHERIT -#define WSA_FLAG_NO_HANDLE_INHERIT 0x80 -#endif - Py_BEGIN_ALLOW_THREADS fd = WSASocketW(family, type, proto, NULL, 0, @@ -6150,8 +6164,9 @@ socket_dup(PyObject *self, PyObject *fdobj) #endif fd = PyLong_AsSocket_t(fdobj); - if (fd == (SOCKET_T)(-1) && PyErr_Occurred()) + if (fd == (SOCKET_T)(-1) && PyErr_Occurred()) { return NULL; + } #ifdef MS_WINDOWS if (WSADuplicateSocketW(fd, GetCurrentProcessId(), &info)) @@ -6160,8 +6175,9 @@ socket_dup(PyObject *self, PyObject *fdobj) newfd = WSASocketW(FROM_PROTOCOL_INFO, FROM_PROTOCOL_INFO, FROM_PROTOCOL_INFO, &info, 0, WSA_FLAG_OVERLAPPED); - if (newfd == INVALID_SOCKET) + if (newfd == INVALID_SOCKET) { return set_error(); + } if (!SetHandleInformation((HANDLE)newfd, HANDLE_FLAG_INHERIT, 0)) { closesocket(newfd); @@ -6171,13 +6187,15 @@ socket_dup(PyObject *self, PyObject *fdobj) #else /* On UNIX, dup can be used to duplicate the file descriptor of a socket */ newfd = _Py_dup(fd); - if (newfd == INVALID_SOCKET) + if (newfd == INVALID_SOCKET) { return NULL; + } #endif newfdobj = PyLong_FromSocket_t(newfd); - if (newfdobj == NULL) + if (newfdobj == NULL) { SOCKETCLOSE(newfd); + } return newfdobj; } diff --git a/Modules/timemodule.c b/Modules/timemodule.c index c2bacaae0c03395..c50e689bb6986ca 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -30,7 +30,9 @@ # include #else # ifdef MS_WINDOWS -# define WIN32_LEAN_AND_MEAN +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif # include # endif /* MS_WINDOWS */ #endif /* !__WATCOMC__ || __QNX__ */ @@ -1135,7 +1137,9 @@ time_tzset(PyObject *self, PyObject *unused) return NULL; } +#if !defined(MS_WINDOWS) || defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) tzset(); +#endif /* Reset timezone, altzone, daylight and tzname */ if (init_timezone(m) < 0) { @@ -1753,7 +1757,9 @@ init_timezone(PyObject *m) */ #ifdef HAVE_DECL_TZNAME PyObject *otz0, *otz1; +#if !defined(MS_WINDOWS) || defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) tzset(); +#endif PyModule_AddIntConstant(m, "timezone", _Py_timezone); #ifdef HAVE_ALTZONE PyModule_AddIntConstant(m, "altzone", altzone); diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 276c5a276c06e60..5e1bcda1d976bb2 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -8,7 +8,6 @@ #include // malloc() #include - #undef uint #define uint pymem_uint diff --git a/PC/clinic/msvcrtmodule.c.h b/PC/clinic/msvcrtmodule.c.h index d808ef0bbd0ffe4..b708c6cdde757c6 100644 --- a/PC/clinic/msvcrtmodule.c.h +++ b/PC/clinic/msvcrtmodule.c.h @@ -261,6 +261,8 @@ msvcrt_getch(PyObject *module, PyObject *Py_UNUSED(ignored)) return return_value; } +#if defined(MS_WINDOWS_DESKTOP) + PyDoc_STRVAR(msvcrt_getwch__doc__, "getwch($module, /)\n" "--\n" @@ -285,6 +287,8 @@ msvcrt_getwch(PyObject *module, PyObject *Py_UNUSED(ignored)) return return_value; } +#endif /* defined(MS_WINDOWS_DESKTOP) */ + PyDoc_STRVAR(msvcrt_getche__doc__, "getche($module, /)\n" "--\n" @@ -309,6 +313,8 @@ msvcrt_getche(PyObject *module, PyObject *Py_UNUSED(ignored)) return return_value; } +#if defined(MS_WINDOWS_DESKTOP) + PyDoc_STRVAR(msvcrt_getwche__doc__, "getwche($module, /)\n" "--\n" @@ -333,6 +339,8 @@ msvcrt_getwche(PyObject *module, PyObject *Py_UNUSED(ignored)) return return_value; } +#endif /* defined(MS_WINDOWS_DESKTOP) */ + PyDoc_STRVAR(msvcrt_putch__doc__, "putch($module, char, /)\n" "--\n" @@ -367,6 +375,8 @@ msvcrt_putch(PyObject *module, PyObject *arg) return return_value; } +#if defined(MS_WINDOWS_DESKTOP) + PyDoc_STRVAR(msvcrt_putwch__doc__, "putwch($module, unicode_char, /)\n" "--\n" @@ -403,6 +413,8 @@ msvcrt_putwch(PyObject *module, PyObject *arg) return return_value; } +#endif /* defined(MS_WINDOWS_DESKTOP) */ + PyDoc_STRVAR(msvcrt_ungetch__doc__, "ungetch($module, char, /)\n" "--\n" @@ -441,6 +453,8 @@ msvcrt_ungetch(PyObject *module, PyObject *arg) return return_value; } +#if defined(MS_WINDOWS_DESKTOP) + PyDoc_STRVAR(msvcrt_ungetwch__doc__, "ungetwch($module, unicode_char, /)\n" "--\n" @@ -477,6 +491,8 @@ msvcrt_ungetwch(PyObject *module, PyObject *arg) return return_value; } +#endif /* defined(MS_WINDOWS_DESKTOP) */ + #if defined(_DEBUG) PyDoc_STRVAR(msvcrt_CrtSetReportFile__doc__, @@ -610,6 +626,8 @@ msvcrt_set_error_mode(PyObject *module, PyObject *arg) #endif /* defined(_DEBUG) */ +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(msvcrt_GetErrorMode__doc__, "GetErrorMode($module, /)\n" "--\n" @@ -628,6 +646,8 @@ msvcrt_GetErrorMode(PyObject *module, PyObject *Py_UNUSED(ignored)) return msvcrt_GetErrorMode_impl(module); } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM)) */ + PyDoc_STRVAR(msvcrt_SetErrorMode__doc__, "SetErrorMode($module, mode, /)\n" "--\n" @@ -656,6 +676,22 @@ msvcrt_SetErrorMode(PyObject *module, PyObject *arg) return return_value; } +#ifndef MSVCRT_GETWCH_METHODDEF + #define MSVCRT_GETWCH_METHODDEF +#endif /* !defined(MSVCRT_GETWCH_METHODDEF) */ + +#ifndef MSVCRT_GETWCHE_METHODDEF + #define MSVCRT_GETWCHE_METHODDEF +#endif /* !defined(MSVCRT_GETWCHE_METHODDEF) */ + +#ifndef MSVCRT_PUTWCH_METHODDEF + #define MSVCRT_PUTWCH_METHODDEF +#endif /* !defined(MSVCRT_PUTWCH_METHODDEF) */ + +#ifndef MSVCRT_UNGETWCH_METHODDEF + #define MSVCRT_UNGETWCH_METHODDEF +#endif /* !defined(MSVCRT_UNGETWCH_METHODDEF) */ + #ifndef MSVCRT_CRTSETREPORTFILE_METHODDEF #define MSVCRT_CRTSETREPORTFILE_METHODDEF #endif /* !defined(MSVCRT_CRTSETREPORTFILE_METHODDEF) */ @@ -667,4 +703,8 @@ msvcrt_SetErrorMode(PyObject *module, PyObject *arg) #ifndef MSVCRT_SET_ERROR_MODE_METHODDEF #define MSVCRT_SET_ERROR_MODE_METHODDEF #endif /* !defined(MSVCRT_SET_ERROR_MODE_METHODDEF) */ -/*[clinic end generated code: output=204bae9fee7f6124 input=a9049054013a1b77]*/ + +#ifndef MSVCRT_GETERRORMODE_METHODDEF + #define MSVCRT_GETERRORMODE_METHODDEF +#endif /* !defined(MSVCRT_GETERRORMODE_METHODDEF) */ +/*[clinic end generated code: output=2db6197608a6aab3 input=a9049054013a1b77]*/ diff --git a/PC/clinic/winreg.c.h b/PC/clinic/winreg.c.h index 2834d9967a77260..7a9474301da8a1f 100644 --- a/PC/clinic/winreg.c.h +++ b/PC/clinic/winreg.c.h @@ -8,6 +8,8 @@ preserve #endif +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_HKEYType_Close__doc__, "Close($self, /)\n" "--\n" @@ -28,6 +30,10 @@ winreg_HKEYType_Close(PyHKEYObject *self, PyObject *Py_UNUSED(ignored)) return winreg_HKEYType_Close_impl(self); } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_HKEYType_Detach__doc__, "Detach($self, /)\n" "--\n" @@ -54,6 +60,10 @@ winreg_HKEYType_Detach(PyHKEYObject *self, PyObject *Py_UNUSED(ignored)) return winreg_HKEYType_Detach_impl(self); } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_HKEYType___enter____doc__, "__enter__($self, /)\n" "--\n" @@ -77,6 +87,10 @@ winreg_HKEYType___enter__(PyHKEYObject *self, PyObject *Py_UNUSED(ignored)) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_HKEYType___exit____doc__, "__exit__($self, /, exc_type, exc_value, traceback)\n" "--\n" @@ -136,6 +150,10 @@ winreg_HKEYType___exit__(PyHKEYObject *self, PyObject *const *args, Py_ssize_t n return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_CloseKey__doc__, "CloseKey($module, hkey, /)\n" "--\n" @@ -151,6 +169,10 @@ PyDoc_STRVAR(winreg_CloseKey__doc__, #define WINREG_CLOSEKEY_METHODDEF \ {"CloseKey", (PyCFunction)winreg_CloseKey, METH_O, winreg_CloseKey__doc__}, +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_ConnectRegistry__doc__, "ConnectRegistry($module, computer_name, key, /)\n" "--\n" @@ -213,6 +235,10 @@ winreg_ConnectRegistry(PyObject *module, PyObject *const *args, Py_ssize_t nargs return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_CreateKey__doc__, "CreateKey($module, key, sub_key, /)\n" "--\n" @@ -278,6 +304,10 @@ winreg_CreateKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_CreateKeyEx__doc__, "CreateKeyEx($module, /, key, sub_key, reserved=0,\n" " access=winreg.KEY_WRITE)\n" @@ -398,6 +428,10 @@ winreg_CreateKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_DeleteKey__doc__, "DeleteKey($module, key, sub_key, /)\n" "--\n" @@ -452,6 +486,10 @@ winreg_DeleteKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_DeleteKeyEx__doc__, "DeleteKeyEx($module, /, key, sub_key, access=winreg.KEY_WOW64_64KEY,\n" " reserved=0)\n" @@ -565,6 +603,10 @@ winreg_DeleteKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_DeleteValue__doc__, "DeleteValue($module, key, value, /)\n" "--\n" @@ -617,6 +659,10 @@ winreg_DeleteValue(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_EnumKey__doc__, "EnumKey($module, key, index, /)\n" "--\n" @@ -661,6 +707,10 @@ winreg_EnumKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_EnumValue__doc__, "EnumValue($module, key, index, /)\n" "--\n" @@ -714,6 +764,10 @@ winreg_EnumValue(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_ExpandEnvironmentStrings__doc__, "ExpandEnvironmentStrings($module, string, /)\n" "--\n" @@ -750,6 +804,10 @@ winreg_ExpandEnvironmentStrings(PyObject *module, PyObject *arg) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_FlushKey__doc__, "FlushKey($module, key, /)\n" "--\n" @@ -790,6 +848,10 @@ winreg_FlushKey(PyObject *module, PyObject *arg) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_LoadKey__doc__, "LoadKey($module, key, sub_key, file_name, /)\n" "--\n" @@ -866,6 +928,10 @@ winreg_LoadKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_OpenKey__doc__, "OpenKey($module, /, key, sub_key, reserved=0, access=winreg.KEY_READ)\n" "--\n" @@ -979,6 +1045,10 @@ winreg_OpenKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_OpenKeyEx__doc__, "OpenKeyEx($module, /, key, sub_key, reserved=0, access=winreg.KEY_READ)\n" "--\n" @@ -1092,6 +1162,10 @@ winreg_OpenKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_QueryInfoKey__doc__, "QueryInfoKey($module, key, /)\n" "--\n" @@ -1128,6 +1202,10 @@ winreg_QueryInfoKey(PyObject *module, PyObject *arg) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_QueryValue__doc__, "QueryValue($module, key, sub_key, /)\n" "--\n" @@ -1189,6 +1267,10 @@ winreg_QueryValue(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_QueryValueEx__doc__, "QueryValueEx($module, key, name, /)\n" "--\n" @@ -1246,6 +1328,10 @@ winreg_QueryValueEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_SaveKey__doc__, "SaveKey($module, key, file_name, /)\n" "--\n" @@ -1303,6 +1389,10 @@ winreg_SaveKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_SetValue__doc__, "SetValue($module, key, sub_key, type, value, /)\n" "--\n" @@ -1384,6 +1474,10 @@ winreg_SetValue(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_SetValueEx__doc__, "SetValueEx($module, key, value_name, reserved, type, value, /)\n" "--\n" @@ -1478,6 +1572,10 @@ winreg_SetValueEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_DisableReflectionKey__doc__, "DisableReflectionKey($module, key, /)\n" "--\n" @@ -1514,6 +1612,10 @@ winreg_DisableReflectionKey(PyObject *module, PyObject *arg) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_EnableReflectionKey__doc__, "EnableReflectionKey($module, key, /)\n" "--\n" @@ -1548,6 +1650,10 @@ winreg_EnableReflectionKey(PyObject *module, PyObject *arg) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_QueryReflectionKey__doc__, "QueryReflectionKey($module, key, /)\n" "--\n" @@ -1579,4 +1685,114 @@ winreg_QueryReflectionKey(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=7e817dc5edc914d3 input=a9049054013a1b77]*/ + +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#ifndef WINREG_HKEYTYPE_CLOSE_METHODDEF + #define WINREG_HKEYTYPE_CLOSE_METHODDEF +#endif /* !defined(WINREG_HKEYTYPE_CLOSE_METHODDEF) */ + +#ifndef WINREG_HKEYTYPE_DETACH_METHODDEF + #define WINREG_HKEYTYPE_DETACH_METHODDEF +#endif /* !defined(WINREG_HKEYTYPE_DETACH_METHODDEF) */ + +#ifndef WINREG_HKEYTYPE___ENTER___METHODDEF + #define WINREG_HKEYTYPE___ENTER___METHODDEF +#endif /* !defined(WINREG_HKEYTYPE___ENTER___METHODDEF) */ + +#ifndef WINREG_HKEYTYPE___EXIT___METHODDEF + #define WINREG_HKEYTYPE___EXIT___METHODDEF +#endif /* !defined(WINREG_HKEYTYPE___EXIT___METHODDEF) */ + +#ifndef WINREG_CLOSEKEY_METHODDEF + #define WINREG_CLOSEKEY_METHODDEF +#endif /* !defined(WINREG_CLOSEKEY_METHODDEF) */ + +#ifndef WINREG_CONNECTREGISTRY_METHODDEF + #define WINREG_CONNECTREGISTRY_METHODDEF +#endif /* !defined(WINREG_CONNECTREGISTRY_METHODDEF) */ + +#ifndef WINREG_CREATEKEY_METHODDEF + #define WINREG_CREATEKEY_METHODDEF +#endif /* !defined(WINREG_CREATEKEY_METHODDEF) */ + +#ifndef WINREG_CREATEKEYEX_METHODDEF + #define WINREG_CREATEKEYEX_METHODDEF +#endif /* !defined(WINREG_CREATEKEYEX_METHODDEF) */ + +#ifndef WINREG_DELETEKEY_METHODDEF + #define WINREG_DELETEKEY_METHODDEF +#endif /* !defined(WINREG_DELETEKEY_METHODDEF) */ + +#ifndef WINREG_DELETEKEYEX_METHODDEF + #define WINREG_DELETEKEYEX_METHODDEF +#endif /* !defined(WINREG_DELETEKEYEX_METHODDEF) */ + +#ifndef WINREG_DELETEVALUE_METHODDEF + #define WINREG_DELETEVALUE_METHODDEF +#endif /* !defined(WINREG_DELETEVALUE_METHODDEF) */ + +#ifndef WINREG_ENUMKEY_METHODDEF + #define WINREG_ENUMKEY_METHODDEF +#endif /* !defined(WINREG_ENUMKEY_METHODDEF) */ + +#ifndef WINREG_ENUMVALUE_METHODDEF + #define WINREG_ENUMVALUE_METHODDEF +#endif /* !defined(WINREG_ENUMVALUE_METHODDEF) */ + +#ifndef WINREG_EXPANDENVIRONMENTSTRINGS_METHODDEF + #define WINREG_EXPANDENVIRONMENTSTRINGS_METHODDEF +#endif /* !defined(WINREG_EXPANDENVIRONMENTSTRINGS_METHODDEF) */ + +#ifndef WINREG_FLUSHKEY_METHODDEF + #define WINREG_FLUSHKEY_METHODDEF +#endif /* !defined(WINREG_FLUSHKEY_METHODDEF) */ + +#ifndef WINREG_LOADKEY_METHODDEF + #define WINREG_LOADKEY_METHODDEF +#endif /* !defined(WINREG_LOADKEY_METHODDEF) */ + +#ifndef WINREG_OPENKEY_METHODDEF + #define WINREG_OPENKEY_METHODDEF +#endif /* !defined(WINREG_OPENKEY_METHODDEF) */ + +#ifndef WINREG_OPENKEYEX_METHODDEF + #define WINREG_OPENKEYEX_METHODDEF +#endif /* !defined(WINREG_OPENKEYEX_METHODDEF) */ + +#ifndef WINREG_QUERYINFOKEY_METHODDEF + #define WINREG_QUERYINFOKEY_METHODDEF +#endif /* !defined(WINREG_QUERYINFOKEY_METHODDEF) */ + +#ifndef WINREG_QUERYVALUE_METHODDEF + #define WINREG_QUERYVALUE_METHODDEF +#endif /* !defined(WINREG_QUERYVALUE_METHODDEF) */ + +#ifndef WINREG_QUERYVALUEEX_METHODDEF + #define WINREG_QUERYVALUEEX_METHODDEF +#endif /* !defined(WINREG_QUERYVALUEEX_METHODDEF) */ + +#ifndef WINREG_SAVEKEY_METHODDEF + #define WINREG_SAVEKEY_METHODDEF +#endif /* !defined(WINREG_SAVEKEY_METHODDEF) */ + +#ifndef WINREG_SETVALUE_METHODDEF + #define WINREG_SETVALUE_METHODDEF +#endif /* !defined(WINREG_SETVALUE_METHODDEF) */ + +#ifndef WINREG_SETVALUEEX_METHODDEF + #define WINREG_SETVALUEEX_METHODDEF +#endif /* !defined(WINREG_SETVALUEEX_METHODDEF) */ + +#ifndef WINREG_DISABLEREFLECTIONKEY_METHODDEF + #define WINREG_DISABLEREFLECTIONKEY_METHODDEF +#endif /* !defined(WINREG_DISABLEREFLECTIONKEY_METHODDEF) */ + +#ifndef WINREG_ENABLEREFLECTIONKEY_METHODDEF + #define WINREG_ENABLEREFLECTIONKEY_METHODDEF +#endif /* !defined(WINREG_ENABLEREFLECTIONKEY_METHODDEF) */ + +#ifndef WINREG_QUERYREFLECTIONKEY_METHODDEF + #define WINREG_QUERYREFLECTIONKEY_METHODDEF +#endif /* !defined(WINREG_QUERYREFLECTIONKEY_METHODDEF) */ +/*[clinic end generated code: output=715db416dc1321ee input=a9049054013a1b77]*/ diff --git a/PC/config.c b/PC/config.c index b1481d79e6508d0..9d0fe6f87df69a7 100644 --- a/PC/config.c +++ b/PC/config.c @@ -43,10 +43,14 @@ extern PyObject* PyInit__collections(void); extern PyObject* PyInit__heapq(void); extern PyObject* PyInit__bisect(void); extern PyObject* PyInit__symtable(void); +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_GAMES) extern PyObject* PyInit_mmap(void); +#endif extern PyObject* PyInit__csv(void); extern PyObject* PyInit__sre(void); +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES) extern PyObject* PyInit_winreg(void); +#endif extern PyObject* PyInit__struct(void); extern PyObject* PyInit__datetime(void); extern PyObject* PyInit__functools(void); @@ -122,10 +126,14 @@ struct _inittab _PyImport_Inittab[] = { {"itertools", PyInit_itertools}, {"_collections", PyInit__collections}, {"_symtable", PyInit__symtable}, +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_GAMES) {"mmap", PyInit_mmap}, +#endif {"_csv", PyInit__csv}, {"_sre", PyInit__sre}, +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES) {"winreg", PyInit_winreg}, +#endif {"_struct", PyInit__struct}, {"_datetime", PyInit__datetime}, {"_functools", PyInit__functools}, diff --git a/PC/config_minimal.c b/PC/config_minimal.c index 928a4efd32e132f..9a66ea1d1cd3482 100644 --- a/PC/config_minimal.c +++ b/PC/config_minimal.c @@ -5,8 +5,10 @@ #include "Python.h" +#ifdef Py_ENABLE_SHARED /* Define extern variables omitted from minimal builds */ void *PyWin_DLLhModule = NULL; +#endif extern PyObject* PyInit_faulthandler(void); @@ -14,7 +16,9 @@ extern PyObject* PyInit__tracemalloc(void); extern PyObject* PyInit_gc(void); extern PyObject* PyInit_nt(void); extern PyObject* PyInit__signal(void); +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES) extern PyObject* PyInit_winreg(void); +#endif extern PyObject* PyInit__ast(void); extern PyObject* PyInit__io(void); @@ -35,7 +39,9 @@ struct _inittab _PyImport_Inittab[] = { {"_tokenize", PyInit__tokenize}, {"_tracemalloc", PyInit__tracemalloc}, +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES) {"winreg", PyInit_winreg}, +#endif /* This module "lives in" with marshal.c */ {"marshal", PyMarshal_Init}, diff --git a/PC/msvcrtmodule.c b/PC/msvcrtmodule.c index 988d9c95aaa22ea..face4d03af9d4f6 100644 --- a/PC/msvcrtmodule.c +++ b/PC/msvcrtmodule.c @@ -253,6 +253,8 @@ msvcrt_getch_impl(PyObject *module) return ch; } +#ifdef MS_WINDOWS_DESKTOP + /*[clinic input] msvcrt.getwch -> wchar_t @@ -271,6 +273,8 @@ msvcrt_getwch_impl(PyObject *module) return ch; } +#endif /* MS_WINDOWS_DESKTOP */ + /*[clinic input] msvcrt.getche -> byte_char @@ -289,6 +293,8 @@ msvcrt_getche_impl(PyObject *module) return ch; } +#ifdef MS_WINDOWS_DESKTOP + /*[clinic input] msvcrt.getwche -> wchar_t @@ -307,6 +313,8 @@ msvcrt_getwche_impl(PyObject *module) return ch; } +#endif /* MS_WINDOWS_DESKTOP */ + /*[clinic input] msvcrt.putch @@ -326,6 +334,8 @@ msvcrt_putch_impl(PyObject *module, char char_value) Py_RETURN_NONE; } +#ifdef MS_WINDOWS_DESKTOP + /*[clinic input] msvcrt.putwch @@ -346,6 +356,8 @@ msvcrt_putwch_impl(PyObject *module, int unicode_char) } +#endif /* MS_WINDOWS_DESKTOP */ + /*[clinic input] msvcrt.ungetch @@ -374,6 +386,8 @@ msvcrt_ungetch_impl(PyObject *module, char char_value) Py_RETURN_NONE; } +#ifdef MS_WINDOWS_DESKTOP + /*[clinic input] msvcrt.ungetwch @@ -398,6 +412,8 @@ msvcrt_ungetwch_impl(PyObject *module, int unicode_char) Py_RETURN_NONE; } +#endif /* MS_WINDOWS_DESKTOP */ + #ifdef _DEBUG /*[clinic input] msvcrt.CrtSetReportFile -> HANDLE @@ -475,6 +491,8 @@ msvcrt_set_error_mode_impl(PyObject *module, int mode) } #endif /* _DEBUG */ +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM) + /*[clinic input] msvcrt.GetErrorMode @@ -494,6 +512,8 @@ msvcrt_GetErrorMode_impl(PyObject *module) return PyLong_FromUnsignedLong(res); } +#endif /* MS_WINDOWS_APP || MS_WINDOWS_SYSTEM */ + /*[clinic input] msvcrt.SetErrorMode @@ -601,10 +621,12 @@ PyInit_msvcrt(void) insertint(d, "LK_NBRLCK", _LK_NBRLCK); insertint(d, "LK_RLCK", _LK_RLCK); insertint(d, "LK_UNLCK", _LK_UNLCK); +#ifdef MS_WINDOWS_DESKTOP insertint(d, "SEM_FAILCRITICALERRORS", SEM_FAILCRITICALERRORS); insertint(d, "SEM_NOALIGNMENTFAULTEXCEPT", SEM_NOALIGNMENTFAULTEXCEPT); insertint(d, "SEM_NOGPFAULTERRORBOX", SEM_NOGPFAULTERRORBOX); insertint(d, "SEM_NOOPENFILEERRORBOX", SEM_NOOPENFILEERRORBOX); +#endif #ifdef _DEBUG insertint(d, "CRT_WARN", _CRT_WARN); insertint(d, "CRT_ERROR", _CRT_ERROR); diff --git a/PC/pyconfig.h b/PC/pyconfig.h index a34d420ab7ecaab..8a3bf8968ce29de 100644 --- a/PC/pyconfig.h +++ b/PC/pyconfig.h @@ -72,9 +72,27 @@ WIN32 is still required for the locale module. #define USE_SOCKET #endif -#if defined(WINAPI_FAMILY) && (WINAPI_FAMILY != WINAPI_FAMILY_DESKTOP_APP) -#define MS_WINDOWS_NON_DESKTOP +#if defined(Py_BUILD_CORE) || defined(Py_BUILD_CORE_BUILTIN) || defined(Py_BUILD_CORE_MODULE) +#include + +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) +#define MS_WINDOWS_DESKTOP +#endif +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP) +#define MS_WINDOWS_APP +#endif +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_SYSTEM) +#define MS_WINDOWS_SYSTEM #endif +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_GAMES) +#define MS_WINDOWS_GAMES +#endif + +/* Define to 1 if you support windows console io */ +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM) +#define HAVE_WINDOWS_CONSOLE_IO 1 +#endif +#endif /* Py_BUILD_CORE || Py_BUILD_CORE_BUILTIN || Py_BUILD_CORE_MODULE */ /* Compiler specific defines */ @@ -300,7 +318,7 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ # endif /* Py_BUILD_CORE */ #endif /* MS_COREDLL */ -#if defined(MS_WIN64) +#ifdef MS_WIN64 /* maintain "win32" sys.platform for backward compatibility of Python code, the Win64 API should be close enough to the Win32 API to make this preferable */ diff --git a/PC/winreg.c b/PC/winreg.c index 86efed09855b01b..073598a12a68aaa 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -18,6 +18,8 @@ #include "structmember.h" // PyMemberDef #include +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES) + static BOOL PyHKEY_AsHKEY(PyObject *ob, HKEY *pRes, BOOL bNoneOK); static BOOL clinic_HKEY_converter(PyObject *ob, void *p); static PyObject *PyHKEY_FromHKEY(HKEY h); @@ -829,6 +831,8 @@ winreg_CloseKey(PyObject *module, PyObject *hkey) Py_RETURN_NONE; } +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) + /*[clinic input] winreg.ConnectRegistry -> HKEY @@ -866,6 +870,8 @@ winreg_ConnectRegistry_impl(PyObject *module, return retKey; } +#endif /* MS_WINDOWS_DESKTOP || MS_WINDOWS_SYSTEM */ + /*[clinic input] winreg.CreateKey -> HKEY @@ -1272,6 +1278,8 @@ winreg_ExpandEnvironmentStrings_impl(PyObject *module, return o; } +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) + /*[clinic input] winreg.FlushKey @@ -1305,6 +1313,9 @@ winreg_FlushKey_impl(PyObject *module, HKEY key) Py_RETURN_NONE; } +#endif /* MS_WINDOWS_DESKTOP || MS_WINDOWS_SYSTEM */ + +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) /*[clinic input] winreg.LoadKey @@ -1354,6 +1365,8 @@ winreg_LoadKey_impl(PyObject *module, HKEY key, const Py_UNICODE *sub_key, Py_RETURN_NONE; } +#endif /* MS_WINDOWS_DESKTOP || MS_WINDOWS_SYSTEM */ + /*[clinic input] winreg.OpenKey -> HKEY @@ -1463,6 +1476,7 @@ winreg_QueryInfoKey_impl(PyObject *module, HKEY key) return ret; } + /*[clinic input] winreg.QueryValue @@ -1634,6 +1648,8 @@ winreg_QueryValueEx_impl(PyObject *module, HKEY key, const Py_UNICODE *name) return result; } +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) + /*[clinic input] winreg.SaveKey @@ -1679,6 +1695,8 @@ winreg_SaveKey_impl(PyObject *module, HKEY key, const Py_UNICODE *file_name) Py_RETURN_NONE; } +#endif /* MS_WINDOWS_DESKTOP || MS_WINDOWS_SYSTEM */ + /*[clinic input] winreg.SetValue @@ -1776,6 +1794,7 @@ winreg_SetValue_impl(PyObject *module, HKEY key, const Py_UNICODE *sub_key, return result; } + /*[clinic input] winreg.SetValueEx @@ -1861,6 +1880,8 @@ winreg_SetValueEx_impl(PyObject *module, HKEY key, return result; } +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) + /*[clinic input] winreg.DisableReflectionKey @@ -2009,6 +2030,8 @@ winreg_QueryReflectionKey_impl(PyObject *module, HKEY key) return PyBool_FromLong(result); } +#endif /* MS_WINDOWS_DESKTOP || MS_WINDOWS_SYSTEM */ + static struct PyMethodDef winreg_methods[] = { WINREG_CLOSEKEY_METHODDEF WINREG_CONNECTREGISTRY_METHODDEF @@ -2149,3 +2172,5 @@ PyMODINIT_FUNC PyInit_winreg(void) ADD_INT(REG_RESOURCE_REQUIREMENTS_LIST); return m; } + +#endif /* MS_WINDOWS_DESKTOP || MS_WINDOWS_SYSTEM || MS_WINDOWS_GAMES */ diff --git a/Parser/myreadline.c b/Parser/myreadline.c index d55fcefbb6f206e..3f0e29f051a4387 100644 --- a/Parser/myreadline.c +++ b/Parser/myreadline.c @@ -13,7 +13,9 @@ #include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH #include "pycore_pystate.h" // _PyThreadState_GET() #ifdef MS_WINDOWS +# ifndef WIN32_LEAN_AND_MEAN # define WIN32_LEAN_AND_MEAN +# endif # include "windows.h" #endif /* MS_WINDOWS */ @@ -108,7 +110,7 @@ my_fgets(PyThreadState* tstate, char *buf, int len, FILE *fp) /* NOTREACHED */ } -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO /* Readline implementation using ReadConsoleW */ extern char _get_console_type(HANDLE handle); @@ -233,7 +235,7 @@ _PyOS_WindowsConsoleReadline(PyThreadState *tstate, HANDLE hStdIn) return buf; } -#endif +#endif /* HAVE_WINDOWS_CONSOLE_IO */ /* Readline implementation using fgets() */ @@ -246,7 +248,7 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt) PyThreadState *tstate = _PyOS_ReadlineTState; assert(tstate != NULL); -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); if (!config->legacy_windows_stdio && sys_stdin == stdin) { HANDLE hStdIn, hStdErr; diff --git a/Python/dynload_win.c b/Python/dynload_win.c index 7bd04d573df4adf..acab05e2c6def3a 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -163,6 +163,7 @@ static char *GetPythonImport (HINSTANCE hModule) return NULL; } +#ifdef Py_ENABLE_SHARED /* Load python3.dll before loading any extension module that might refer to it. That way, we can be sure that always the python3.dll corresponding to this python DLL is loaded, not a python3.dll that might be on the path @@ -216,6 +217,7 @@ _Py_CheckPython3(void) return hPython3 != NULL; #undef MAXPATHLEN } +#endif /* Py_ENABLE_SHARED */ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, const char *shortname, @@ -224,7 +226,9 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, dl_funcptr p; char funcname[258], *import_python; +#ifdef Py_ENABLE_SHARED _Py_CheckPython3(); +#endif /* Py_ENABLE_SHARED */ wchar_t *wpathname = PyUnicode_AsWideCharString(pathname, NULL); if (wpathname == NULL) @@ -234,10 +238,12 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, { HINSTANCE hDLL = NULL; +#ifdef MS_WINDOWS_DESKTOP unsigned int old_mode; /* Don't display a message box when Python can't load a DLL */ old_mode = SetErrorMode(SEM_FAILCRITICALERRORS); +#endif /* bpo-36085: We use LoadLibraryEx with restricted search paths to avoid DLL preloading attacks and enable use of the @@ -250,8 +256,10 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, Py_END_ALLOW_THREADS PyMem_Free(wpathname); +#ifdef MS_WINDOWS_DESKTOP /* restore old error mode settings */ SetErrorMode(old_mode); +#endif if (hDLL==NULL){ PyObject *message; diff --git a/Python/fileutils.c b/Python/fileutils.c index 93bee9ee007cd4d..4ac759c45a3a1ee 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -8,7 +8,11 @@ #ifdef MS_WINDOWS # include # include -# include // PathCchCombineEx +# if defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) +# define PATHCCH_ALLOW_LONG_PATHS 0x01 +# else +# include // PathCchCombineEx +# endif extern int winerror_to_errno(int); #endif @@ -77,7 +81,8 @@ _Py_device_encoding(int fd) if (!valid) Py_RETURN_NONE; -#if defined(MS_WINDOWS) +#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO UINT cp; if (fd == 0) cp = GetConsoleCP(); @@ -92,6 +97,9 @@ _Py_device_encoding(int fd) } return PyUnicode_FromFormat("cp%u", (unsigned int)cp); +#else + Py_RETURN_NONE; +#endif /* HAVE_WINDOWS_CONSOLE_IO */ #else if (_PyRuntime.preconfig.utf8_mode) { _Py_DECLARE_STR(utf_8, "utf-8"); @@ -1270,6 +1278,13 @@ _Py_stat(PyObject *path, struct stat *statbuf) #endif } +#ifdef MS_WINDOWS +// For some Windows API partitions, SetHandleInformation() is declared +// but none of the handle flags are defined. +#ifndef HANDLE_FLAG_INHERIT +#define HANDLE_FLAG_INHERIT 0x00000001 +#endif +#endif /* This function MUST be kept async-signal-safe on POSIX when raise=0. */ static int @@ -2096,6 +2111,72 @@ _Py_abspath(const wchar_t *path, wchar_t **abspath_p) #endif } +// The Windows Games API family implements the PathCch* APIs in the Xbox OS, +// but does not expose them yet. Load them dynamically until +// 1) they are officially exposed +// 2) we stop supporting older versions of the GDK which do not expose them +#if defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) +HRESULT +PathCchSkipRoot(const wchar_t *path, const wchar_t **rootEnd) +{ + static int initialized = 0; + typedef HRESULT(__stdcall *PPathCchSkipRoot) (PCWSTR pszPath, + PCWSTR *ppszRootEnd); + static PPathCchSkipRoot _PathCchSkipRoot; + + if (initialized == 0) { + HMODULE pathapi = LoadLibraryExW(L"api-ms-win-core-path-l1-1-0.dll", NULL, + LOAD_LIBRARY_SEARCH_SYSTEM32); + if (pathapi) { + _PathCchSkipRoot = (PPathCchSkipRoot)GetProcAddress( + pathapi, "PathCchSkipRoot"); + } + else { + _PathCchSkipRoot = NULL; + } + initialized = 1; + } + + if (!_PathCchSkipRoot) { + return E_NOINTERFACE; + } + + return _PathCchSkipRoot(path, rootEnd); +} + +static HRESULT +PathCchCombineEx(wchar_t *buffer, size_t bufsize, const wchar_t *dirname, + const wchar_t *relfile, unsigned long flags) +{ + static int initialized = 0; + typedef HRESULT(__stdcall *PPathCchCombineEx) (PWSTR pszPathOut, + size_t cchPathOut, + PCWSTR pszPathIn, + PCWSTR pszMore, + unsigned long dwFlags); + static PPathCchCombineEx _PathCchCombineEx; + + if (initialized == 0) { + HMODULE pathapi = LoadLibraryExW(L"api-ms-win-core-path-l1-1-0.dll", NULL, + LOAD_LIBRARY_SEARCH_SYSTEM32); + if (pathapi) { + _PathCchCombineEx = (PPathCchCombineEx)GetProcAddress( + pathapi, "PathCchCombineEx"); + } + else { + _PathCchCombineEx = NULL; + } + initialized = 1; + } + + if (!_PathCchCombineEx) { + return E_NOINTERFACE; + } + + return _PathCchCombineEx(buffer, bufsize, dirname, relfile, flags); +} + +#endif /* defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) */ // The caller must ensure "buffer" is big enough. static int @@ -2491,12 +2572,12 @@ _Py_get_blocking(int fd) success = GetNamedPipeHandleStateW(handle, &mode, NULL, NULL, NULL, NULL, 0); Py_END_ALLOW_THREADS - + if (!success) { PyErr_SetFromWindowsErr(0); return -1; } - + return !(mode & PIPE_NOWAIT); } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index e80dd30c89dfd0c..82e94090a6027ad 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2289,7 +2289,7 @@ create_stdio(const PyConfig *config, PyObject* io, raw = Py_NewRef(buf); } -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO /* Windows console IO is always UTF-8 encoded */ PyTypeObject *winconsoleio_type = (PyTypeObject *)_PyImport_GetModuleAttr( &_Py_ID(_io), &_Py_ID(_WindowsConsoleIO)); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 207abb964bcac93..764fb70bae6c384 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1490,6 +1490,9 @@ static PyStructSequence_Desc windows_version_desc = { static PyObject * _sys_getwindowsversion_from_kernel32() { +#ifndef MS_WINDOWS_DESKTOP + return NULL; +#else HANDLE hKernel32; wchar_t kernel32_path[MAX_PATH]; LPVOID verblock; @@ -1523,6 +1526,7 @@ _sys_getwindowsversion_from_kernel32() realBuild = HIWORD(ffi->dwProductVersionLS); PyMem_RawFree(verblock); return Py_BuildValue("(kkk)", realMajor, realMinor, realBuild); +#endif /* !MS_WINDOWS_DESKTOP */ } /* Disable deprecation warnings about GetVersionEx as the result is From 71cf7c3dddd9c49ec70c1a95547f2fcd5daa7034 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Thu, 9 Mar 2023 23:06:20 +0000 Subject: [PATCH 50/97] gh-102378: don't bother stripping `/` from __text_signature__ (#102379) --- Lib/inspect.py | 57 ++++++------------- Lib/test/test_inspect.py | 23 +++----- ...-03-03-19-53-08.gh-issue-102378.kRdOZc.rst | 1 + 3 files changed, 25 insertions(+), 56 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-03-19-53-08.gh-issue-102378.kRdOZc.rst diff --git a/Lib/inspect.py b/Lib/inspect.py index 166667c62cdccf8..edc23b0ffa9201f 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2106,26 +2106,21 @@ def _signature_strip_non_python_syntax(signature): Private helper function. Takes a signature in Argument Clinic's extended signature format. - Returns a tuple of three things: - * that signature re-rendered in standard Python syntax, + Returns a tuple of two things: + * that signature re-rendered in standard Python syntax, and * the index of the "self" parameter (generally 0), or None if - the function does not have a "self" parameter, and - * the index of the last "positional only" parameter, - or None if the signature has no positional-only parameters. + the function does not have a "self" parameter. """ if not signature: - return signature, None, None + return signature, None self_parameter = None - last_positional_only = None lines = [l.encode('ascii') for l in signature.split('\n') if l] generator = iter(lines).__next__ token_stream = tokenize.tokenize(generator) - delayed_comma = False - skip_next_comma = False text = [] add = text.append @@ -2142,35 +2137,18 @@ def _signature_strip_non_python_syntax(signature): if type == OP: if string == ',': - if skip_next_comma: - skip_next_comma = False - else: - assert not delayed_comma - delayed_comma = True - current_parameter += 1 - continue - - if string == '/': - assert not skip_next_comma - assert last_positional_only is None - skip_next_comma = True - last_positional_only = current_parameter - 1 - continue + current_parameter += 1 if (type == ERRORTOKEN) and (string == '$'): assert self_parameter is None self_parameter = current_parameter continue - if delayed_comma: - delayed_comma = False - if not ((type == OP) and (string == ')')): - add(', ') add(string) if (string == ','): add(' ') clean_signature = ''.join(text) - return clean_signature, self_parameter, last_positional_only + return clean_signature, self_parameter def _signature_fromstr(cls, obj, s, skip_bound_arg=True): @@ -2179,8 +2157,7 @@ def _signature_fromstr(cls, obj, s, skip_bound_arg=True): """ Parameter = cls._parameter_cls - clean_signature, self_parameter, last_positional_only = \ - _signature_strip_non_python_syntax(s) + clean_signature, self_parameter = _signature_strip_non_python_syntax(s) program = "def foo" + clean_signature + ": pass" @@ -2269,17 +2246,17 @@ def p(name_node, default_node, default=empty): parameters.append(Parameter(name, kind, default=default, annotation=empty)) # non-keyword-only parameters - args = reversed(f.args.args) - defaults = reversed(f.args.defaults) - iter = itertools.zip_longest(args, defaults, fillvalue=None) - if last_positional_only is not None: - kind = Parameter.POSITIONAL_ONLY - else: - kind = Parameter.POSITIONAL_OR_KEYWORD - for i, (name, default) in enumerate(reversed(list(iter))): + total_non_kw_args = len(f.args.posonlyargs) + len(f.args.args) + required_non_kw_args = total_non_kw_args - len(f.args.defaults) + defaults = itertools.chain(itertools.repeat(None, required_non_kw_args), f.args.defaults) + + kind = Parameter.POSITIONAL_ONLY + for (name, default) in zip(f.args.posonlyargs, defaults): + p(name, default) + + kind = Parameter.POSITIONAL_OR_KEYWORD + for (name, default) in zip(f.args.args, defaults): p(name, default) - if i == last_positional_only: - kind = Parameter.POSITIONAL_OR_KEYWORD # *args if f.args.vararg: diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 92aba519d28a085..02f8378d0413eac 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -4230,56 +4230,47 @@ def foo(a): pass class TestSignaturePrivateHelpers(unittest.TestCase): def _strip_non_python_syntax(self, input, - clean_signature, self_parameter, last_positional_only): + clean_signature, self_parameter): computed_clean_signature, \ - computed_self_parameter, \ - computed_last_positional_only = \ + computed_self_parameter = \ inspect._signature_strip_non_python_syntax(input) self.assertEqual(computed_clean_signature, clean_signature) self.assertEqual(computed_self_parameter, self_parameter) - self.assertEqual(computed_last_positional_only, last_positional_only) def test_signature_strip_non_python_syntax(self): self._strip_non_python_syntax( "($module, /, path, mode, *, dir_fd=None, " + "effective_ids=False,\n follow_symlinks=True)", - "(module, path, mode, *, dir_fd=None, " + + "(module, /, path, mode, *, dir_fd=None, " + "effective_ids=False, follow_symlinks=True)", - 0, 0) self._strip_non_python_syntax( "($module, word, salt, /)", - "(module, word, salt)", - 0, - 2) + "(module, word, salt, /)", + 0) self._strip_non_python_syntax( "(x, y=None, z=None, /)", - "(x, y=None, z=None)", - None, - 2) + "(x, y=None, z=None, /)", + None) self._strip_non_python_syntax( "(x, y=None, z=None)", "(x, y=None, z=None)", - None, None) self._strip_non_python_syntax( "(x,\n y=None,\n z = None )", "(x, y=None, z=None)", - None, None) self._strip_non_python_syntax( "", "", - None, None) self._strip_non_python_syntax( - None, None, None, None) diff --git a/Misc/NEWS.d/next/Library/2023-03-03-19-53-08.gh-issue-102378.kRdOZc.rst b/Misc/NEWS.d/next/Library/2023-03-03-19-53-08.gh-issue-102378.kRdOZc.rst new file mode 100644 index 000000000000000..d30f65f30d109aa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-03-19-53-08.gh-issue-102378.kRdOZc.rst @@ -0,0 +1 @@ +Private helper method ``inspect._signature_strip_non_python_syntax`` will no longer strip ``/`` from the input string. From 2999e02836f9112de6b17784eaca762fb87e71a9 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Fri, 10 Mar 2023 09:02:32 +0000 Subject: [PATCH 51/97] gh-102192: Replace PyErr_Fetch/Restore etc by more efficient alternatives in `_ctypes` (#102477) --- Modules/_ctypes/_ctypes.c | 11 ++--------- Modules/_ctypes/callproc.c | 36 +++++++++++++++++++----------------- 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 8690f2c1b078524..6f92ca08dd537b7 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -2200,7 +2200,6 @@ PyCSimpleType_from_param(PyObject *type, PyObject *value) struct fielddesc *fd; PyObject *as_parameter; int res; - PyObject *exc, *val, *tb; /* If the value is already an instance of the requested type, we can use it as is */ @@ -2234,33 +2233,27 @@ PyCSimpleType_from_param(PyObject *type, PyObject *value) parg->obj = fd->setfunc(&parg->value, value, 0); if (parg->obj) return (PyObject *)parg; - PyErr_Fetch(&exc, &val, &tb); + PyObject *exc = PyErr_GetRaisedException(); Py_DECREF(parg); if (_PyObject_LookupAttr(value, &_Py_ID(_as_parameter_), &as_parameter) < 0) { Py_XDECREF(exc); - Py_XDECREF(val); - Py_XDECREF(tb); return NULL; } if (as_parameter) { if (_Py_EnterRecursiveCall("while processing _as_parameter_")) { Py_DECREF(as_parameter); Py_XDECREF(exc); - Py_XDECREF(val); - Py_XDECREF(tb); return NULL; } value = PyCSimpleType_from_param(type, as_parameter); _Py_LeaveRecursiveCall(); Py_DECREF(as_parameter); Py_XDECREF(exc); - Py_XDECREF(val); - Py_XDECREF(tb); return value; } if (exc) { - PyErr_Restore(exc, val, tb); + PyErr_SetRaisedException(exc); } else { PyErr_SetString(PyExc_TypeError, "wrong type"); diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index f6d98bbeebc24e9..4438727332bc11a 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1013,41 +1013,43 @@ static PyObject *GetResult(PyObject *restype, void *result, PyObject *checker) void _ctypes_extend_error(PyObject *exc_class, const char *fmt, ...) { va_list vargs; - PyObject *tp, *v, *tb, *s, *cls_str, *msg_str; va_start(vargs, fmt); - s = PyUnicode_FromFormatV(fmt, vargs); + PyObject *s = PyUnicode_FromFormatV(fmt, vargs); va_end(vargs); - if (!s) + if (s == NULL) { return; + } - PyErr_Fetch(&tp, &v, &tb); - PyErr_NormalizeException(&tp, &v, &tb); - if (PyType_Check(tp)) - cls_str = PyType_GetName((PyTypeObject *)tp); - else - cls_str = PyObject_Str(tp); + assert(PyErr_Occurred()); + PyObject *exc = PyErr_GetRaisedException(); + assert(exc != NULL); + PyObject *cls_str = PyType_GetName(Py_TYPE(exc)); if (cls_str) { PyUnicode_AppendAndDel(&s, cls_str); PyUnicode_AppendAndDel(&s, PyUnicode_FromString(": ")); - if (s == NULL) + if (s == NULL) { goto error; - } else + } + } + else { PyErr_Clear(); - msg_str = PyObject_Str(v); - if (msg_str) + } + + PyObject *msg_str = PyObject_Str(exc); + if (msg_str) { PyUnicode_AppendAndDel(&s, msg_str); + } else { PyErr_Clear(); PyUnicode_AppendAndDel(&s, PyUnicode_FromString("???")); } - if (s == NULL) + if (s == NULL) { goto error; + } PyErr_SetObject(exc_class, s); error: - Py_XDECREF(tp); - Py_XDECREF(v); - Py_XDECREF(tb); + Py_XDECREF(exc); Py_XDECREF(s); } From cb35882773a3ffc7fe0671e64848f4c926a2d52f Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 10 Mar 2023 12:21:37 +0000 Subject: [PATCH 52/97] gh-102519: Add os.listdrives, os.listvolumes and os.listmounts on Windows (GH-102544) --- Doc/whatsnew/3.12.rst | 4 + .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 2 + Lib/test/test_os.py | 43 ++++ ...-03-08-23-08-38.gh-issue-102519.wlcsFI.rst | 2 + Modules/clinic/posixmodule.c.h | 128 +++++++++++- Modules/posixmodule.c | 194 ++++++++++++++++++ 9 files changed, 375 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-08-23-08-38.gh-issue-102519.wlcsFI.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index d982cb62ec2f4e3..48b7aab0595ebb2 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -294,6 +294,10 @@ os method to check if the entry is a junction. (Contributed by Charles Machalow in :gh:`99547`.) +* Add :func:`os.listdrives`, :func:`os.listvolumes` and :func:`os.listmounts` + functions on Windows for enumerating drives, volumes and mount points. + (Contributed by Steve Dower in :gh:`102519`.) + os.path ------- diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index dc5cd58d853534b..4b12ae523c32601 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1216,6 +1216,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(values)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(version)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(volume)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(warnings)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(warnoptions)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(wbits)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 8b23aa154793018..17fb9ffbbf9f112 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -702,6 +702,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(value) STRUCT_FOR_ID(values) STRUCT_FOR_ID(version) + STRUCT_FOR_ID(volume) STRUCT_FOR_ID(warnings) STRUCT_FOR_ID(warnoptions) STRUCT_FOR_ID(wbits) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 471efadb13bb4f4..b240be57369d9df 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1208,6 +1208,7 @@ extern "C" { INIT_ID(value), \ INIT_ID(values), \ INIT_ID(version), \ + INIT_ID(volume), \ INIT_ID(warnings), \ INIT_ID(warnoptions), \ INIT_ID(wbits), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index b47d240e492ff97..52af37a8e60aa8e 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1310,6 +1310,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(version); PyUnicode_InternInPlace(&string); + string = &_Py_ID(volume); + PyUnicode_InternInPlace(&string); string = &_Py_ID(warnings); PyUnicode_InternInPlace(&string); string = &_Py_ID(warnoptions); diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index ba6feb69ea17125..253e2a23238f12a 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2648,6 +2648,49 @@ def test_listdir_extended_path(self): [os.fsencode(path) for path in self.created_paths]) +@unittest.skipUnless(os.name == "nt", "NT specific tests") +class Win32ListdriveTests(unittest.TestCase): + """Test listdrive, listmounts and listvolume on Windows.""" + + def setUp(self): + # Get drives and volumes from fsutil + out = subprocess.check_output( + ["fsutil.exe", "volume", "list"], + cwd=os.path.join(os.getenv("SystemRoot", "\\Windows"), "System32"), + encoding="mbcs", + errors="ignore", + ) + lines = out.splitlines() + self.known_volumes = {l for l in lines if l.startswith('\\\\?\\')} + self.known_drives = {l for l in lines if l[1:] == ':\\'} + self.known_mounts = {l for l in lines if l[1:3] == ':\\'} + + def test_listdrives(self): + drives = os.listdrives() + self.assertIsInstance(drives, list) + self.assertSetEqual( + self.known_drives, + self.known_drives & set(drives), + ) + + def test_listvolumes(self): + volumes = os.listvolumes() + self.assertIsInstance(volumes, list) + self.assertSetEqual( + self.known_volumes, + self.known_volumes & set(volumes), + ) + + def test_listmounts(self): + for volume in os.listvolumes(): + mounts = os.listmounts(volume) + self.assertIsInstance(mounts, list) + self.assertSetEqual( + set(mounts), + self.known_mounts & set(mounts), + ) + + @unittest.skipUnless(hasattr(os, 'readlink'), 'needs os.readlink()') class ReadlinkTests(unittest.TestCase): filelink = 'readlinktest' diff --git a/Misc/NEWS.d/next/Library/2023-03-08-23-08-38.gh-issue-102519.wlcsFI.rst b/Misc/NEWS.d/next/Library/2023-03-08-23-08-38.gh-issue-102519.wlcsFI.rst new file mode 100644 index 000000000000000..f47e4f70b1301d4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-08-23-08-38.gh-issue-102519.wlcsFI.rst @@ -0,0 +1,2 @@ +Add :func:`os.listdrives`, :func:`os.listvolumes` and :func:`os.listmounts` +functions on Windows for enumerating drives, volumes and mount points diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 6565f8df935cb85..8b0550d832fc0aa 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -1601,6 +1601,120 @@ os_listdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #if defined(MS_WINDOWS) +PyDoc_STRVAR(os_listdrives__doc__, +"listdrives($module, /)\n" +"--\n" +"\n" +"Return a list containing the names of drives in the system.\n" +"\n" +"A drive name typically looks like \'C:\\\\\'."); + +#define OS_LISTDRIVES_METHODDEF \ + {"listdrives", (PyCFunction)os_listdrives, METH_NOARGS, os_listdrives__doc__}, + +static PyObject * +os_listdrives_impl(PyObject *module); + +static PyObject * +os_listdrives(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return os_listdrives_impl(module); +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(os_listvolumes__doc__, +"listvolumes($module, /)\n" +"--\n" +"\n" +"Return a list containing the volumes in the system.\n" +"\n" +"Volumes are typically represented as a GUID path."); + +#define OS_LISTVOLUMES_METHODDEF \ + {"listvolumes", (PyCFunction)os_listvolumes, METH_NOARGS, os_listvolumes__doc__}, + +static PyObject * +os_listvolumes_impl(PyObject *module); + +static PyObject * +os_listvolumes(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return os_listvolumes_impl(module); +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(os_listmounts__doc__, +"listmounts($module, /, volume)\n" +"--\n" +"\n" +"Return a list containing mount points for a particular volume.\n" +"\n" +"\'volume\' should be a GUID path as returned from os.listvolumes."); + +#define OS_LISTMOUNTS_METHODDEF \ + {"listmounts", _PyCFunction_CAST(os_listmounts), METH_FASTCALL|METH_KEYWORDS, os_listmounts__doc__}, + +static PyObject * +os_listmounts_impl(PyObject *module, path_t *volume); + +static PyObject * +os_listmounts(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(volume), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"volume", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "listmounts", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + path_t volume = PATH_T_INITIALIZE("listmounts", "volume", 0, 0); + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!path_converter(args[0], &volume)) { + goto exit; + } + return_value = os_listmounts_impl(module, &volume); + +exit: + /* Cleanup for volume */ + path_cleanup(&volume); + + return return_value; +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + PyDoc_STRVAR(os__getfullpathname__doc__, "_getfullpathname($module, path, /)\n" "--\n" @@ -11253,6 +11367,18 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #define OS_LINK_METHODDEF #endif /* !defined(OS_LINK_METHODDEF) */ +#ifndef OS_LISTDRIVES_METHODDEF + #define OS_LISTDRIVES_METHODDEF +#endif /* !defined(OS_LISTDRIVES_METHODDEF) */ + +#ifndef OS_LISTVOLUMES_METHODDEF + #define OS_LISTVOLUMES_METHODDEF +#endif /* !defined(OS_LISTVOLUMES_METHODDEF) */ + +#ifndef OS_LISTMOUNTS_METHODDEF + #define OS_LISTMOUNTS_METHODDEF +#endif /* !defined(OS_LISTMOUNTS_METHODDEF) */ + #ifndef OS__GETFULLPATHNAME_METHODDEF #define OS__GETFULLPATHNAME_METHODDEF #endif /* !defined(OS__GETFULLPATHNAME_METHODDEF) */ @@ -11796,4 +11922,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=9495478e51701b8a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=47750e0e29c8d707 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 0d534f35f6a420a..7d91f7e4bac76b9 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -4229,7 +4229,198 @@ os_listdir_impl(PyObject *module, path_t *path) #endif } + #ifdef MS_WINDOWS + +/*[clinic input] +os.listdrives + +Return a list containing the names of drives in the system. + +A drive name typically looks like 'C:\\'. + +[clinic start generated code]*/ + +static PyObject * +os_listdrives_impl(PyObject *module) +/*[clinic end generated code: output=aaece9dacdf682b5 input=1af9ccc9e583798e]*/ +{ + /* Number of possible drives is limited, so 256 should always be enough. + On the day when it is not, listmounts() will have to be used. */ + wchar_t buffer[256]; + DWORD buflen = Py_ARRAY_LENGTH(buffer); + PyObject *result = NULL; + if (PySys_Audit("os.listdrives", NULL) < 0) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS; + buflen = GetLogicalDriveStringsW(buflen, buffer); + Py_END_ALLOW_THREADS; + + if (!buflen) { + PyErr_SetFromWindowsErr(0); + return NULL; + } else if (buflen >= Py_ARRAY_LENGTH(buffer)) { + PyErr_SetFromWindowsErr(ERROR_MORE_DATA); + return NULL; + } + + /* buflen includes a null terminator, so remove it */ + PyObject *str = PyUnicode_FromWideChar(buffer, buflen - 1); + if (str) { + PyObject *nullchar = PyUnicode_FromStringAndSize("\0", 1); + if (nullchar) { + result = PyUnicode_Split(str, nullchar, -1); + Py_DECREF(nullchar); + } + Py_DECREF(str); + } + return result; +} + +/*[clinic input] +os.listvolumes + +Return a list containing the volumes in the system. + +Volumes are typically represented as a GUID path. + +[clinic start generated code]*/ + +static PyObject * +os_listvolumes_impl(PyObject *module) +/*[clinic end generated code: output=534e10ea2bf9d386 input=f6e4e70371f11e99]*/ +{ + PyObject *result = PyList_New(0); + HANDLE find = INVALID_HANDLE_VALUE; + wchar_t buffer[MAX_PATH + 1]; + if (!result) { + return NULL; + } + if (PySys_Audit("os.listvolumes", NULL) < 0) { + Py_DECREF(result); + return NULL; + } + + int err = 0; + Py_BEGIN_ALLOW_THREADS; + find = FindFirstVolumeW(buffer, Py_ARRAY_LENGTH(buffer)); + if (find == INVALID_HANDLE_VALUE) { + err = GetLastError(); + } + Py_END_ALLOW_THREADS; + + while (!err) { + PyObject *s = PyUnicode_FromWideChar(buffer, -1); + if (!s || PyList_Append(result, s) < 0) { + Py_XDECREF(s); + Py_CLEAR(result); + break; + } + Py_DECREF(s); + + Py_BEGIN_ALLOW_THREADS; + if (!FindNextVolumeW(find, buffer, Py_ARRAY_LENGTH(buffer))) { + err = GetLastError(); + } + Py_END_ALLOW_THREADS; + } + + if (find != INVALID_HANDLE_VALUE) { + Py_BEGIN_ALLOW_THREADS; + FindVolumeClose(find); + Py_END_ALLOW_THREADS; + } + if (err && err != ERROR_NO_MORE_FILES) { + PyErr_SetFromWindowsErr(err); + Py_XDECREF(result); + result = NULL; + } + return result; +} + + +/*[clinic input] +os.listmounts + + volume: path_t + +Return a list containing mount points for a particular volume. + +'volume' should be a GUID path as returned from os.listvolumes. + +[clinic start generated code]*/ + +static PyObject * +os_listmounts_impl(PyObject *module, path_t *volume) +/*[clinic end generated code: output=06da49679de4512e input=a8a27178e3f67845]*/ +{ + wchar_t default_buffer[MAX_PATH + 1]; + DWORD buflen = Py_ARRAY_LENGTH(default_buffer); + LPWSTR buffer = default_buffer; + DWORD attributes; + PyObject *str = NULL; + PyObject *nullchar = NULL; + PyObject *result = NULL; + + /* Ensure we have a valid volume path before continuing */ + Py_BEGIN_ALLOW_THREADS + attributes = GetFileAttributesW(volume->wide); + Py_END_ALLOW_THREADS + if (attributes == INVALID_FILE_ATTRIBUTES && + GetLastError() == ERROR_UNRECOGNIZED_VOLUME) + { + return PyErr_SetFromWindowsErr(ERROR_UNRECOGNIZED_VOLUME); + } + + if (PySys_Audit("os.listmounts", "O", volume->object) < 0) { + return NULL; + } + + while (1) { + BOOL success; + Py_BEGIN_ALLOW_THREADS + success = GetVolumePathNamesForVolumeNameW(volume->wide, buffer, + buflen, &buflen); + Py_END_ALLOW_THREADS + if (success) { + break; + } + if (GetLastError() != ERROR_MORE_DATA) { + PyErr_SetFromWindowsErr(0); + goto exit; + } + if (buffer != default_buffer) { + PyMem_Free((void *)buffer); + } + buffer = (wchar_t*)PyMem_Malloc(sizeof(wchar_t) * buflen); + if (!buffer) { + PyErr_NoMemory(); + goto exit; + } + } + if (buflen < 2) { + result = PyList_New(0); + goto exit; + } + // buflen includes two null terminators, one for the last string + // and one for the array of strings. + str = PyUnicode_FromWideChar(buffer, buflen - 2); + nullchar = PyUnicode_FromStringAndSize("\0", 1); + if (str && nullchar) { + result = PyUnicode_Split(str, nullchar, -1); + } +exit: + if (buffer != default_buffer) { + PyMem_Free(buffer); + } + Py_XDECREF(nullchar); + Py_XDECREF(str); + return result; +} + + int _PyOS_getfullpathname(const wchar_t *path, wchar_t **abspath_p) { @@ -15252,6 +15443,9 @@ static PyMethodDef posix_methods[] = { OS_GETCWDB_METHODDEF OS_LINK_METHODDEF OS_LISTDIR_METHODDEF + OS_LISTDRIVES_METHODDEF + OS_LISTMOUNTS_METHODDEF + OS_LISTVOLUMES_METHODDEF OS_LSTAT_METHODDEF OS_MKDIR_METHODDEF OS_NICE_METHODDEF From 53dceb53ade15587b9cfd30c0a0942232517dee9 Mon Sep 17 00:00:00 2001 From: Owain Davies <116417456+OTheDev@users.noreply.github.com> Date: Fri, 10 Mar 2023 19:22:02 +0700 Subject: [PATCH 53/97] gh-86509: Add link to Lib/_threading_local.py in threading docs (#101824) --- Doc/library/threading.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index b352125551fa79e..83ed48052704fb9 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -272,7 +272,7 @@ The instance's values will be different for separate threads. A class that represents thread-local data. For more details and extensive examples, see the documentation string of the - :mod:`_threading_local` module. + :mod:`_threading_local` module: :source:`Lib/_threading_local.py`. .. _thread-objects: From 64bde502cf89963bc7382b03ea9e1c0967d22e35 Mon Sep 17 00:00:00 2001 From: Paul Ganssle <1377457+pganssle@users.noreply.github.com> Date: Fri, 10 Mar 2023 10:29:37 -0500 Subject: [PATCH 54/97] GH-102537: Handle check for PYTHONTZPATH failing in zoneinfo test (GH-102538) It is possible but unlikely for the `python_tzpath_context` function to fail between the start of the `try` block and the point where `os.environ.get` succeeds, in which case `old_env` will be undefined. In this case, we want to take no action. Practically speaking this will really only happen in an error condition anyway, so it doesn't really matter, but we should probably do it right anyway. --- Lib/test/test_zoneinfo/test_zoneinfo.py | 9 ++++++++- .../Tests/2023-03-08-13-54-20.gh-issue-102537.Vfplpb.rst | 2 ++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-03-08-13-54-20.gh-issue-102537.Vfplpb.rst diff --git a/Lib/test/test_zoneinfo/test_zoneinfo.py b/Lib/test/test_zoneinfo/test_zoneinfo.py index 82041a2b488334f..ae921f7432c4665 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo.py @@ -1543,13 +1543,20 @@ class TzPathTest(TzPathUserMixin, ZoneInfoTestBase): @contextlib.contextmanager def python_tzpath_context(value): path_var = "PYTHONTZPATH" + unset_env_sentinel = object() + old_env = unset_env_sentinel try: with OS_ENV_LOCK: old_env = os.environ.get(path_var, None) os.environ[path_var] = value yield finally: - if old_env is None: + if old_env is unset_env_sentinel: + # In this case, `old_env` was never retrieved from the + # environment for whatever reason, so there's no need to + # reset the environment TZPATH. + pass + elif old_env is None: del os.environ[path_var] else: os.environ[path_var] = old_env # pragma: nocover diff --git a/Misc/NEWS.d/next/Tests/2023-03-08-13-54-20.gh-issue-102537.Vfplpb.rst b/Misc/NEWS.d/next/Tests/2023-03-08-13-54-20.gh-issue-102537.Vfplpb.rst new file mode 100644 index 000000000000000..94d160dd4127a6c --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-03-08-13-54-20.gh-issue-102537.Vfplpb.rst @@ -0,0 +1,2 @@ +Adjust the error handling strategy in +``test_zoneinfo.TzPathTest.python_tzpath_context``. Patch by Paul Ganssle. From 12226bec2588f925f4698e1130ce78e118343934 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 10 Mar 2023 15:41:32 +0000 Subject: [PATCH 55/97] gh-102519: Add doc updates for os.listdrives, listvolumes and listmounts (GH-102585) --- Doc/library/os.rst | 63 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 23ce98785bedfc6..5b9f49be1fad55f 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -2188,6 +2188,69 @@ features: Accepts a :term:`path-like object`. +.. function:: listdrives() + + Return a list containing the names of drives on a Windows system. + + A drive name typically looks like ``'C:\\'``. Not every drive name + will be associated with a volume, and some may be inaccessible for + a variety of reasons, including permissions, network connectivity + or missing media. This function does not test for access. + + May raise :exc:`OSError` if an error occurs collecting the drive + names. + + .. audit-event:: os.listdrives "" os.listdrives + + .. availability:: Windows + + .. versionadded:: 3.12 + + +.. function:: listmounts(volume) + + Return a list containing the mount points for a volume on a Windows + system. + + *volume* must be represented as a GUID path, like those returned by + :func:`os.listvolumes`. Volumes may be mounted in multiple locations + or not at all. In the latter case, the list will be empty. Mount + points that are not associated with a volume will not be returned by + this function. + + The mount points return by this function will be absolute paths, and + may be longer than the drive name. + + Raises :exc:`OSError` if the volume is not recognized or if an error + occurs collecting the paths. + + .. audit-event:: os.listmounts volume os.listmounts + + .. availability:: Windows + + .. versionadded:: 3.12 + + +.. function:: listvolumes() + + Return a list containing the volumes in the system. + + Volumes are typically represented as a GUID path that looks like + ``\\?\Volume{xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx}\``. Files can + usually be accessed through a GUID path, permissions allowing. + However, users are generally not familiar with them, and so the + recommended use of this function is to retrieve mount points + using :func:`os.listmounts`. + + May raise :exc:`OSError` if an error occurs collecting the volumes. + + .. audit-event:: os.listvolumes "" os.listvolumes + + .. availability:: Windows + + .. versionadded:: 3.12 + + .. function:: lstat(path, *, dir_fd=None) Perform the equivalent of an :c:func:`lstat` system call on the given path. From 90f1d777177e28b6c7b8d9ba751550e373d61b0a Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Fri, 10 Mar 2023 17:29:04 +0000 Subject: [PATCH 56/97] GH-80486: Fix handling of NTFS alternate data streams in pathlib (GH-102454) Co-authored-by: Maor Kleinberger --- Lib/pathlib.py | 8 ++++-- Lib/test/test_pathlib.py | 28 ++++++++++++++++++- .../2019-03-15-22-50-27.bpo-36305.Pbkv6u.rst | 2 ++ 3 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2019-03-15-22-50-27.bpo-36305.Pbkv6u.rst diff --git a/Lib/pathlib.py b/Lib/pathlib.py index d375529ff5f7676..55c44f12e5a2fba 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -320,8 +320,9 @@ def _from_parsed_parts(cls, drv, root, parts): def _format_parsed_parts(cls, drv, root, parts): if drv or root: return drv + root + cls._flavour.sep.join(parts[1:]) - else: - return cls._flavour.sep.join(parts) + elif parts and cls._flavour.splitdrive(parts[0])[0]: + parts = ['.'] + parts + return cls._flavour.sep.join(parts) def __str__(self): """Return the string representation of the path, suitable for @@ -1188,7 +1189,8 @@ def expanduser(self): homedir = self._flavour.expanduser(self._parts[0]) if homedir[:1] == "~": raise RuntimeError("Could not determine home directory.") - return self._from_parts([homedir] + self._parts[1:]) + drv, root, parts = self._parse_parts((homedir,)) + return self._from_parsed_parts(drv, root, parts + self._parts[1:]) return self diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index f8e2f44d27fc1ef..f05dead5886743e 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -122,6 +122,13 @@ def test_parse_parts(self): # the second path is relative. check(['c:/a/b', 'c:x/y'], ('c:', '\\', ['c:\\', 'a', 'b', 'x', 'y'])) check(['c:/a/b', 'c:/x/y'], ('c:', '\\', ['c:\\', 'x', 'y'])) + # Paths to files with NTFS alternate data streams + check(['./c:s'], ('', '', ['c:s'])) + check(['cc:s'], ('', '', ['cc:s'])) + check(['C:c:s'], ('C:', '', ['C:', 'c:s'])) + check(['C:/c:s'], ('C:', '\\', ['C:\\', 'c:s'])) + check(['D:a', './c:b'], ('D:', '', ['D:', 'a', 'c:b'])) + check(['D:/a', './c:b'], ('D:', '\\', ['D:\\', 'a', 'c:b'])) # @@ -165,6 +172,7 @@ def test_constructor_common(self): self.assertEqual(P(P('a'), 'b'), P('a/b')) self.assertEqual(P(P('a'), P('b')), P('a/b')) self.assertEqual(P(P('a'), P('b'), P('c')), P(FakePath("a/b/c"))) + self.assertEqual(P(P('./a:b')), P('./a:b')) def test_bytes(self): P = self.cls @@ -814,7 +822,8 @@ class PureWindowsPathTest(_BasePurePathTest, unittest.TestCase): equivalences = _BasePurePathTest.equivalences.copy() equivalences.update({ - 'c:a': [ ('c:', 'a'), ('c:', 'a/'), ('/', 'c:', 'a') ], + './a:b': [ ('./a:b',) ], + 'c:a': [ ('c:', 'a'), ('c:', 'a/'), ('.', 'c:', 'a') ], 'c:/a': [ ('c:/', 'a'), ('c:', '/', 'a'), ('c:', '/a'), ('/z', 'c:/', 'a'), ('//x/y', 'c:/', 'a'), @@ -838,6 +847,7 @@ def test_str(self): self.assertEqual(str(p), '\\\\a\\b\\c\\d') def test_str_subclass(self): + self._check_str_subclass('.\\a:b') self._check_str_subclass('c:') self._check_str_subclass('c:a') self._check_str_subclass('c:a\\b.txt') @@ -1005,6 +1015,7 @@ def test_drive(self): self.assertEqual(P('//a/b').drive, '\\\\a\\b') self.assertEqual(P('//a/b/').drive, '\\\\a\\b') self.assertEqual(P('//a/b/c/d').drive, '\\\\a\\b') + self.assertEqual(P('./c:a').drive, '') def test_root(self): P = self.cls @@ -1341,6 +1352,14 @@ def test_join(self): self.assertEqual(pp, P('C:/a/b/x/y')) pp = p.joinpath('c:/x/y') self.assertEqual(pp, P('C:/x/y')) + # Joining with files with NTFS data streams => the filename should + # not be parsed as a drive letter + pp = p.joinpath(P('./d:s')) + self.assertEqual(pp, P('C:/a/b/d:s')) + pp = p.joinpath(P('./dd:s')) + self.assertEqual(pp, P('C:/a/b/dd:s')) + pp = p.joinpath(P('E:d:s')) + self.assertEqual(pp, P('E:d:s')) def test_div(self): # Basically the same as joinpath(). @@ -1361,6 +1380,11 @@ def test_div(self): # the second path is relative. self.assertEqual(p / 'c:x/y', P('C:/a/b/x/y')) self.assertEqual(p / 'c:/x/y', P('C:/x/y')) + # Joining with files with NTFS data streams => the filename should + # not be parsed as a drive letter + self.assertEqual(p / P('./d:s'), P('C:/a/b/d:s')) + self.assertEqual(p / P('./dd:s'), P('C:/a/b/dd:s')) + self.assertEqual(p / P('E:d:s'), P('E:d:s')) def test_is_reserved(self): P = self.cls @@ -1626,6 +1650,8 @@ def test_expanduser_common(self): self.assertEqual(p.expanduser(), p) p = P(P('').absolute().anchor) / '~' self.assertEqual(p.expanduser(), p) + p = P('~/a:b') + self.assertEqual(p.expanduser(), P(os.path.expanduser('~'), './a:b')) def test_exists(self): P = self.cls diff --git a/Misc/NEWS.d/next/Library/2019-03-15-22-50-27.bpo-36305.Pbkv6u.rst b/Misc/NEWS.d/next/Library/2019-03-15-22-50-27.bpo-36305.Pbkv6u.rst new file mode 100644 index 000000000000000..d9360496ac24cb9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-03-15-22-50-27.bpo-36305.Pbkv6u.rst @@ -0,0 +1,2 @@ +Fix handling of Windows filenames that resemble drives, such as ``./a:b``, +in :mod:`pathlib`. From ee6f8413a99d0ee4828e1c81911e203d3fff85d5 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Sat, 11 Mar 2023 08:21:22 +0800 Subject: [PATCH 57/97] gh-102578: Optimise setting and deleting mutable attributes on non-dataclass subclasses of frozen dataclasses (gh-102573) --- Lib/dataclasses.py | 10 ++--- Lib/test/test_dataclasses.py | 44 +++++++++++++++++++ ...-03-10-13-21-16.gh-issue-102578.-gujoI.rst | 4 ++ 3 files changed, 52 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-10-13-21-16.gh-issue-102578.-gujoI.rst diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 8bc8594d674bc0d..78a126f051e2e76 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -616,21 +616,19 @@ def _repr_fn(fields, globals): def _frozen_get_del_attr(cls, fields, globals): locals = {'cls': cls, 'FrozenInstanceError': FrozenInstanceError} + condition = 'type(self) is cls' if fields: - fields_str = '(' + ','.join(repr(f.name) for f in fields) + ',)' - else: - # Special case for the zero-length tuple. - fields_str = '()' + condition += ' or name in {' + ', '.join(repr(f.name) for f in fields) + '}' return (_create_fn('__setattr__', ('self', 'name', 'value'), - (f'if type(self) is cls or name in {fields_str}:', + (f'if {condition}:', ' raise FrozenInstanceError(f"cannot assign to field {name!r}")', f'super(cls, self).__setattr__(name, value)'), locals=locals, globals=globals), _create_fn('__delattr__', ('self', 'name'), - (f'if type(self) is cls or name in {fields_str}:', + (f'if {condition}:', ' raise FrozenInstanceError(f"cannot delete field {name!r}")', f'super(cls, self).__delattr__(name)'), locals=locals, diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 589f229f4623594..5486b2ef3f47e51 100644 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -2767,6 +2767,19 @@ class C: c.i = 5 self.assertEqual(c.i, 10) + def test_frozen_empty(self): + @dataclass(frozen=True) + class C: + pass + + c = C() + self.assertFalse(hasattr(c, 'i')) + with self.assertRaises(FrozenInstanceError): + c.i = 5 + self.assertFalse(hasattr(c, 'i')) + with self.assertRaises(FrozenInstanceError): + del c.i + def test_inherit(self): @dataclass(frozen=True) class C: @@ -2890,6 +2903,37 @@ class S(D): self.assertEqual(s.y, 10) self.assertEqual(s.cached, True) + with self.assertRaises(FrozenInstanceError): + del s.x + self.assertEqual(s.x, 3) + with self.assertRaises(FrozenInstanceError): + del s.y + self.assertEqual(s.y, 10) + del s.cached + self.assertFalse(hasattr(s, 'cached')) + with self.assertRaises(AttributeError) as cm: + del s.cached + self.assertNotIsInstance(cm.exception, FrozenInstanceError) + + def test_non_frozen_normal_derived_from_empty_frozen(self): + @dataclass(frozen=True) + class D: + pass + + class S(D): + pass + + s = S() + self.assertFalse(hasattr(s, 'x')) + s.x = 5 + self.assertEqual(s.x, 5) + + del s.x + self.assertFalse(hasattr(s, 'x')) + with self.assertRaises(AttributeError) as cm: + del s.x + self.assertNotIsInstance(cm.exception, FrozenInstanceError) + def test_overwriting_frozen(self): # frozen uses __setattr__ and __delattr__. with self.assertRaisesRegex(TypeError, diff --git a/Misc/NEWS.d/next/Library/2023-03-10-13-21-16.gh-issue-102578.-gujoI.rst b/Misc/NEWS.d/next/Library/2023-03-10-13-21-16.gh-issue-102578.-gujoI.rst new file mode 100644 index 000000000000000..7307148d9a81efc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-10-13-21-16.gh-issue-102578.-gujoI.rst @@ -0,0 +1,4 @@ +Speed up setting or deleting mutable attributes on non-dataclass subclasses of +frozen dataclasses. Due to the implementation of ``__setattr__`` and +``__delattr__`` for frozen dataclasses, this previously had a time complexity +of ``O(n)``. It now has a time complexity of ``O(1)``. From b48be8fa18518583abb21bf6e4f5d7e4b5c9d7b2 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sat, 11 Mar 2023 03:26:46 +0300 Subject: [PATCH 58/97] gh-102103: add `module` argument to `dataclasses.make_dataclass` (#102104) --- Doc/library/dataclasses.rst | 6 ++- Lib/dataclasses.py | 15 ++++++- Lib/test/test_dataclasses.py | 39 +++++++++++++++++++ ...-02-21-11-56-16.gh-issue-102103.Dj0WEj.rst | 2 + 4 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-21-11-56-16.gh-issue-102103.Dj0WEj.rst diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 82faa7b77450fb6..5f4dc25bfd78771 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -389,7 +389,7 @@ Module contents :func:`astuple` raises :exc:`TypeError` if ``obj`` is not a dataclass instance. -.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False) +.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False, module=None) Creates a new dataclass with name ``cls_name``, fields as defined in ``fields``, base classes as given in ``bases``, and initialized @@ -401,6 +401,10 @@ Module contents ``match_args``, ``kw_only``, ``slots``, and ``weakref_slot`` have the same meaning as they do in :func:`dataclass`. + If ``module`` is defined, the ``__module__`` attribute + of the dataclass is set to that value. + By default, it is set to the module name of the caller. + This function is not strictly required, because any Python mechanism for creating a new class with ``__annotations__`` can then apply the :func:`dataclass` function to convert that class to diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 78a126f051e2e76..24f3779ebb8ec8e 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1391,7 +1391,7 @@ def _astuple_inner(obj, tuple_factory): def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, - weakref_slot=False): + weakref_slot=False, module=None): """Return a new dynamically created dataclass. The dataclass name will be 'cls_name'. 'fields' is an iterable @@ -1455,6 +1455,19 @@ def exec_body_callback(ns): # of generic dataclasses. cls = types.new_class(cls_name, bases, {}, exec_body_callback) + # For pickling to work, the __module__ variable needs to be set to the frame + # where the dataclass is created. + if module is None: + try: + module = sys._getframemodulename(1) or '__main__' + except AttributeError: + try: + module = sys._getframe(1).f_globals.get('__name__', '__main__') + except (AttributeError, ValueError): + pass + if module is not None: + cls.__module__ = module + # Apply the normal decorator. return dataclass(cls, init=init, repr=repr, eq=eq, order=order, unsafe_hash=unsafe_hash, frozen=frozen, diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 5486b2ef3f47e51..76bed0c33146734 100644 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -3606,6 +3606,15 @@ def test_text_annotations(self): 'return': type(None)}) +ByMakeDataClass = make_dataclass('ByMakeDataClass', [('x', int)]) +ManualModuleMakeDataClass = make_dataclass('ManualModuleMakeDataClass', + [('x', int)], + module='test.test_dataclasses') +WrongNameMakeDataclass = make_dataclass('Wrong', [('x', int)]) +WrongModuleMakeDataclass = make_dataclass('WrongModuleMakeDataclass', + [('x', int)], + module='custom') + class TestMakeDataclass(unittest.TestCase): def test_simple(self): C = make_dataclass('C', @@ -3715,6 +3724,36 @@ def test_no_types(self): 'y': int, 'z': 'typing.Any'}) + def test_module_attr(self): + self.assertEqual(ByMakeDataClass.__module__, __name__) + self.assertEqual(ByMakeDataClass(1).__module__, __name__) + self.assertEqual(WrongModuleMakeDataclass.__module__, "custom") + Nested = make_dataclass('Nested', []) + self.assertEqual(Nested.__module__, __name__) + self.assertEqual(Nested().__module__, __name__) + + def test_pickle_support(self): + for klass in [ByMakeDataClass, ManualModuleMakeDataClass]: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + self.assertEqual( + pickle.loads(pickle.dumps(klass, proto)), + klass, + ) + self.assertEqual( + pickle.loads(pickle.dumps(klass(1), proto)), + klass(1), + ) + + def test_cannot_be_pickled(self): + for klass in [WrongNameMakeDataclass, WrongModuleMakeDataclass]: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + with self.assertRaises(pickle.PickleError): + pickle.dumps(klass, proto) + with self.assertRaises(pickle.PickleError): + pickle.dumps(klass(1), proto) + def test_invalid_type_specification(self): for bad_field in [(), (1, 2, 3, 4), diff --git a/Misc/NEWS.d/next/Library/2023-02-21-11-56-16.gh-issue-102103.Dj0WEj.rst b/Misc/NEWS.d/next/Library/2023-02-21-11-56-16.gh-issue-102103.Dj0WEj.rst new file mode 100644 index 000000000000000..feba433f5bee899 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-21-11-56-16.gh-issue-102103.Dj0WEj.rst @@ -0,0 +1,2 @@ +Add ``module`` argument to :func:`dataclasses.make_dataclass` and make +classes produced by it pickleable. From 767d3a8f6f2f94daa15ad3759d0ecdf4c009b7ab Mon Sep 17 00:00:00 2001 From: "Ilya V. Schurov" Date: Sat, 11 Mar 2023 01:27:41 +0100 Subject: [PATCH 59/97] gh-88071: Update docstrings of dataclass' astuple and asdict (#101806) Update dataclasses.astuple and dataclasses.asdict docstrings to reflect that they deep copy objects in the field values. --- Lib/dataclasses.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 24f3779ebb8ec8e..f4617b1dbdac669 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1281,7 +1281,7 @@ class C: If given, 'dict_factory' will be used instead of built-in dict. The function applies recursively to field values that are dataclass instances. This will also look into built-in containers: - tuples, lists, and dicts. + tuples, lists, and dicts. Other objects are copied with 'copy.deepcopy()'. """ if not _is_dataclass_instance(obj): raise TypeError("asdict() should be called on dataclass instances") @@ -1353,7 +1353,7 @@ class C: If given, 'tuple_factory' will be used instead of built-in tuple. The function applies recursively to field values that are dataclass instances. This will also look into built-in containers: - tuples, lists, and dicts. + tuples, lists, and dicts. Other objects are copied with 'copy.deepcopy()'. """ if not _is_dataclass_instance(obj): From 08b67fb34f4519be1b0bb4673643a2c761c7ae92 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 10 Mar 2023 17:01:16 -0800 Subject: [PATCH 60/97] GH-90997: Shrink the LOAD_GLOBAL caches (#102569) --- Doc/library/dis.rst | 6 +- Include/internal/pycore_code.h | 2 +- Include/internal/pycore_opcode.h | 2 +- Lib/importlib/_bootstrap_external.py | 13 +- Lib/opcode.py | 2 +- Lib/test/test_dis.py | 292 +++++++++--------- ...3-03-09-13-57-35.gh-issue-90997.J-Yhn2.rst | 2 + Python/bytecodes.c | 6 +- Python/generated_cases.c.h | 14 +- Python/opcode_metadata.h | 8 +- Python/specialize.c | 12 +- 11 files changed, 187 insertions(+), 172 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-09-13-57-35.gh-issue-90997.J-Yhn2.rst diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index a5bc5e7fb6ea716..f4f47b3bf4846db 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -57,9 +57,9 @@ the following command can be used to display the disassembly of 2 0 RESUME 0 3 2 LOAD_GLOBAL 1 (NULL + len) - 14 LOAD_FAST 0 (alist) - 16 CALL 1 - 26 RETURN_VALUE + 12 LOAD_FAST 0 (alist) + 14 CALL 1 + 24 RETURN_VALUE (The "2" is a line number). diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 10f1e320a12ff43..6bd212dd42c6f27 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -20,7 +20,7 @@ extern "C" { typedef struct { uint16_t counter; uint16_t index; - uint16_t module_keys_version[2]; + uint16_t module_keys_version; uint16_t builtin_keys_version; } _PyLoadGlobalCache; diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index f9ab95ca4bb9d34..da94fc81dcbe3e7 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -48,7 +48,7 @@ const uint8_t _PyOpcode_Caches[256] = { [STORE_ATTR] = 4, [LOAD_ATTR] = 9, [COMPARE_OP] = 1, - [LOAD_GLOBAL] = 5, + [LOAD_GLOBAL] = 4, [BINARY_OP] = 1, [SEND] = 1, [COMPARE_AND_BRANCH] = 1, diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 954401cfa85ed3d..a01a0955182de5c 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -431,12 +431,17 @@ def _write_atomic(path, data, mode=0o666): # Python 3.12a5 3515 (Embed jump mask in COMPARE_OP oparg) # Python 3.12a5 3516 (Add COMPARE_AND_BRANCH instruction) # Python 3.12a5 3517 (Change YIELD_VALUE oparg to exception block depth) -# Python 3.12a5 3518 (Add RETURN_CONST instruction) -# Python 3.12a5 3519 (Modify SEND instruction) -# Python 3.12a5 3520 (Remove PREP_RERAISE_STAR, add CALL_INTRINSIC_2) +# Python 3.12a6 3518 (Add RETURN_CONST instruction) +# Python 3.12a6 3519 (Modify SEND instruction) +# Python 3.12a6 3520 (Remove PREP_RERAISE_STAR, add CALL_INTRINSIC_2) +# Python 3.12a7 3521 (Shrink the LOAD_GLOBAL caches) # Python 3.13 will start with 3550 +# Please don't copy-paste the same pre-release tag for new entries above!!! +# You should always use the *upcoming* tag. For example, if 3.12a6 came out +# a week ago, I should put "Python 3.12a7" next to my new magic number. + # MAGIC must change whenever the bytecode emitted by the compiler may no # longer be understood by older implementations of the eval loop (usually # due to the addition of new opcodes). @@ -446,7 +451,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3520).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3521).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index 809d24e51676bde..23529d87a09ef94 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -390,7 +390,7 @@ def pseudo_op(name, op, real_ops): "LOAD_GLOBAL": { "counter": 1, "index": 1, - "module_keys_version": 2, + "module_keys_version": 1, "builtin_keys_version": 1, }, "BINARY_OP": { diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 9086824dd6f40cd..b77e3b06d0c1f1e 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -138,10 +138,10 @@ def bug708901(): %3d CALL 2 GET_ITER - >> FOR_ITER 2 (to 38) + >> FOR_ITER 2 (to 36) STORE_FAST 0 (res) -%3d JUMP_BACKWARD 4 (to 30) +%3d JUMP_BACKWARD 4 (to 28) %3d >> END_FOR RETURN_CONST 0 (None) @@ -354,7 +354,7 @@ def bug42562(): %3d LOAD_GLOBAL 0 (Exception) CHECK_EXC_MATCH - POP_JUMP_IF_FALSE 23 (to 82) + POP_JUMP_IF_FALSE 23 (to 80) STORE_FAST 0 (e) %3d LOAD_FAST 0 (e) @@ -730,14 +730,14 @@ def loop_test(): LOAD_CONST 2 (3) BINARY_OP 5 (*) GET_ITER - >> FOR_ITER_LIST 15 (to 50) + >> FOR_ITER_LIST 14 (to 48) STORE_FAST 0 (i) %3d LOAD_GLOBAL_MODULE 1 (NULL + load_test) LOAD_FAST 0 (i) CALL_PY_WITH_DEFAULTS 1 POP_TOP - JUMP_BACKWARD 17 (to 16) + JUMP_BACKWARD 16 (to 16) %3d >> END_FOR RETURN_CONST 0 (None) @@ -1189,8 +1189,8 @@ def test_show_caches(self): caches = list(self.get_cached_values(quickened, adaptive)) for cache in caches: self.assertRegex(cache, pattern) - total_caches = 23 - empty_caches = 8 + total_caches = 22 + empty_caches = 7 self.assertEqual(caches.count(""), empty_caches) self.assertEqual(len(caches), total_caches) @@ -1483,17 +1483,17 @@ def _prepare_test_cases(): Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='defaults, closure', offset=16, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='f', argrepr='f', offset=18, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='NULL + print', offset=20, starts_line=7, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=0, argval='a', argrepr='a', offset=32, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=1, argval='b', argrepr='b', offset=34, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval='', argrepr="''", offset=36, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=1, argrepr='1', offset=38, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BUILD_LIST', opcode=103, arg=0, argval=0, argrepr='', offset=40, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BUILD_MAP', opcode=105, arg=0, argval=0, argrepr='', offset=42, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='Hello world!', argrepr="'Hello world!'", offset=44, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=7, argval=7, argrepr='', offset=46, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=56, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='f', argrepr='f', offset=58, starts_line=8, is_jump_target=False, positions=None), - Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=60, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=137, arg=0, argval='a', argrepr='a', offset=30, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=137, arg=1, argval='b', argrepr='b', offset=32, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval='', argrepr="''", offset=34, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=1, argrepr='1', offset=36, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='BUILD_LIST', opcode=103, arg=0, argval=0, argrepr='', offset=38, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='BUILD_MAP', opcode=105, arg=0, argval=0, argrepr='', offset=40, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='Hello world!', argrepr="'Hello world!'", offset=42, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=7, argval=7, argrepr='', offset=44, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=54, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='f', argrepr='f', offset=56, starts_line=8, is_jump_target=False, positions=None), + Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=58, starts_line=None, is_jump_target=False, positions=None), ] expected_opinfo_f = [ @@ -1511,145 +1511,145 @@ def _prepare_test_cases(): Instruction(opname='MAKE_FUNCTION', opcode=132, arg=9, argval=9, argrepr='defaults, closure', offset=22, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='inner', argrepr='inner', offset=24, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='NULL + print', offset=26, starts_line=5, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='a', argrepr='a', offset=38, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='b', argrepr='b', offset=40, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=0, argval='c', argrepr='c', offset=42, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=1, argval='d', argrepr='d', offset=44, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=4, argval=4, argrepr='', offset=46, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=56, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='inner', argrepr='inner', offset=58, starts_line=6, is_jump_target=False, positions=None), - Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=60, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='a', argrepr='a', offset=36, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='b', argrepr='b', offset=38, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=137, arg=0, argval='c', argrepr='c', offset=40, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=137, arg=1, argval='d', argrepr='d', offset=42, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=4, argval=4, argrepr='', offset=44, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=54, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='inner', argrepr='inner', offset=56, starts_line=6, is_jump_target=False, positions=None), + Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=58, starts_line=None, is_jump_target=False, positions=None), ] expected_opinfo_inner = [ Instruction(opname='COPY_FREE_VARS', opcode=149, arg=4, argval=4, argrepr='', offset=0, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='RESUME', opcode=151, arg=0, argval=0, argrepr='', offset=2, starts_line=3, is_jump_target=False, positions=None), Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='NULL + print', offset=4, starts_line=4, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=2, argval='a', argrepr='a', offset=16, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='b', argrepr='b', offset=18, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='c', argrepr='c', offset=20, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=5, argval='d', argrepr='d', offset=22, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='e', argrepr='e', offset=24, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=1, argval='f', argrepr='f', offset=26, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=6, argval=6, argrepr='', offset=28, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=38, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RETURN_CONST', opcode=121, arg=0, argval=None, argrepr='None', offset=40, starts_line=None, is_jump_target=False, positions=None) + Instruction(opname='LOAD_DEREF', opcode=137, arg=2, argval='a', argrepr='a', offset=14, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='b', argrepr='b', offset=16, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='c', argrepr='c', offset=18, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=137, arg=5, argval='d', argrepr='d', offset=20, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='e', argrepr='e', offset=22, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=1, argval='f', argrepr='f', offset=24, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=6, argval=6, argrepr='', offset=26, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=36, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RETURN_CONST', opcode=121, arg=0, argval=None, argrepr='None', offset=38, starts_line=None, is_jump_target=False, positions=None), ] expected_opinfo_jumpy = [ Instruction(opname='RESUME', opcode=151, arg=0, argval=0, argrepr='', offset=0, starts_line=1, is_jump_target=False, positions=None), Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='range', argrepr='NULL + range', offset=2, starts_line=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=10, argrepr='10', offset=14, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=16, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='GET_ITER', opcode=68, arg=None, argval=None, argrepr='', offset=26, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='FOR_ITER', opcode=93, arg=28, argval=88, argrepr='to 88', offset=28, starts_line=None, is_jump_target=True, positions=None), - Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=32, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=34, starts_line=4, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=46, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=48, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=58, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=60, starts_line=5, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=62, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_AND_BRANCH', opcode=141, arg=13, argval='<', argrepr='<', offset=64, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=72, argrepr='to 72', offset=68, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=22, argval=28, argrepr='to 28', offset=70, starts_line=6, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=72, starts_line=7, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=74, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_AND_BRANCH', opcode=141, arg=68, argval='>', argrepr='>', offset=76, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=115, arg=1, argval=84, argrepr='to 84', offset=80, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=28, argval=28, argrepr='to 28', offset=82, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=84, starts_line=8, is_jump_target=True, positions=None), - Instruction(opname='JUMP_FORWARD', opcode=110, arg=14, argval=116, argrepr='to 116', offset=86, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='END_FOR', opcode=4, arg=None, argval=None, argrepr='', offset=88, starts_line=3, is_jump_target=True, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=90, starts_line=10, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=102, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=104, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=114, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST_CHECK', opcode=127, arg=0, argval='i', argrepr='i', offset=116, starts_line=11, is_jump_target=True, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=33, argval=186, argrepr='to 186', offset=118, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=120, starts_line=12, is_jump_target=True, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=132, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=134, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=144, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=146, starts_line=13, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=148, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BINARY_OP', opcode=122, arg=23, argval=23, argrepr='-=', offset=150, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=154, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=156, starts_line=14, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=158, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_AND_BRANCH', opcode=141, arg=75, argval='>', argrepr='>', offset=160, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=168, argrepr='to 168', offset=164, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=26, argval=116, argrepr='to 116', offset=166, starts_line=15, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=168, starts_line=16, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=170, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_AND_BRANCH', opcode=141, arg=13, argval='<', argrepr='<', offset=172, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=180, argrepr='to 180', offset=176, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_FORWARD', opcode=110, arg=16, argval=212, argrepr='to 212', offset=178, starts_line=17, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=180, starts_line=11, is_jump_target=True, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=186, argrepr='to 186', offset=182, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=33, argval=120, argrepr='to 120', offset=184, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=186, starts_line=19, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=198, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=200, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=210, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='NOP', opcode=9, arg=None, argval=None, argrepr='', offset=212, starts_line=20, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=214, starts_line=21, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval=0, argrepr='0', offset=216, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BINARY_OP', opcode=122, arg=11, argval=11, argrepr='/', offset=218, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=222, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=224, starts_line=25, is_jump_target=False, positions=None), - Instruction(opname='BEFORE_WITH', opcode=53, arg=None, argval=None, argrepr='', offset=226, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='STORE_FAST', opcode=125, arg=1, argval='dodgy', argrepr='dodgy', offset=228, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=230, starts_line=26, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=242, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=244, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=254, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=256, starts_line=25, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=258, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=260, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=2, argval=2, argrepr='', offset=262, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=272, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=274, starts_line=28, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=286, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=288, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=298, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RETURN_CONST', opcode=121, arg=0, argval=None, argrepr='None', offset=300, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=302, starts_line=25, is_jump_target=False, positions=None), - Instruction(opname='WITH_EXCEPT_START', opcode=49, arg=None, argval=None, argrepr='', offset=304, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=115, arg=1, argval=310, argrepr='to 310', offset=306, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=2, argval=2, argrepr='', offset=308, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=310, starts_line=None, is_jump_target=True, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=312, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=314, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=316, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=23, argval=274, argrepr='to 274', offset=318, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=320, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=322, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=324, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=326, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=328, starts_line=22, is_jump_target=False, positions=None), - Instruction(opname='CHECK_EXC_MATCH', opcode=36, arg=None, argval=None, argrepr='', offset=340, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=16, argval=376, argrepr='to 376', offset=342, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=344, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=346, starts_line=23, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=358, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=360, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=370, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=372, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=51, argval=274, argrepr='to 274', offset=374, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=0, argval=0, argrepr='', offset=376, starts_line=22, is_jump_target=True, positions=None), - Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=378, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=380, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=382, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=384, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=386, starts_line=28, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=398, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=400, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=410, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=0, argval=0, argrepr='', offset=412, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=414, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=416, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=418, starts_line=None, is_jump_target=False, positions=None) + Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=10, argrepr='10', offset=12, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=14, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='GET_ITER', opcode=68, arg=None, argval=None, argrepr='', offset=24, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='FOR_ITER', opcode=93, arg=27, argval=84, argrepr='to 84', offset=26, starts_line=None, is_jump_target=True, positions=None), + Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=30, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=32, starts_line=4, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=42, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=44, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=54, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=56, starts_line=5, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=58, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_AND_BRANCH', opcode=141, arg=13, argval='<', argrepr='<', offset=60, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=68, argrepr='to 68', offset=64, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=140, arg=21, argval=26, argrepr='to 26', offset=66, starts_line=6, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=68, starts_line=7, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=70, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_AND_BRANCH', opcode=141, arg=68, argval='>', argrepr='>', offset=72, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=115, arg=1, argval=80, argrepr='to 80', offset=76, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=140, arg=27, argval=26, argrepr='to 26', offset=78, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=80, starts_line=8, is_jump_target=True, positions=None), + Instruction(opname='JUMP_FORWARD', opcode=110, arg=13, argval=110, argrepr='to 110', offset=82, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='END_FOR', opcode=4, arg=None, argval=None, argrepr='', offset=84, starts_line=3, is_jump_target=True, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=86, starts_line=10, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=96, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=98, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=108, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST_CHECK', opcode=127, arg=0, argval='i', argrepr='i', offset=110, starts_line=11, is_jump_target=True, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=32, argval=178, argrepr='to 178', offset=112, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=114, starts_line=12, is_jump_target=True, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=124, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=126, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=136, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=138, starts_line=13, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=140, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='BINARY_OP', opcode=122, arg=23, argval=23, argrepr='-=', offset=142, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=146, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=148, starts_line=14, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=150, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_AND_BRANCH', opcode=141, arg=75, argval='>', argrepr='>', offset=152, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=160, argrepr='to 160', offset=156, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=140, arg=25, argval=110, argrepr='to 110', offset=158, starts_line=15, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=160, starts_line=16, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=162, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_AND_BRANCH', opcode=141, arg=13, argval='<', argrepr='<', offset=164, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=172, argrepr='to 172', offset=168, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_FORWARD', opcode=110, arg=15, argval=202, argrepr='to 202', offset=170, starts_line=17, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=172, starts_line=11, is_jump_target=True, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=178, argrepr='to 178', offset=174, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=140, arg=32, argval=114, argrepr='to 114', offset=176, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=178, starts_line=19, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=188, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=190, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=200, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='NOP', opcode=9, arg=None, argval=None, argrepr='', offset=202, starts_line=20, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=204, starts_line=21, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval=0, argrepr='0', offset=206, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='BINARY_OP', opcode=122, arg=11, argval=11, argrepr='/', offset=208, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=212, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=214, starts_line=25, is_jump_target=False, positions=None), + Instruction(opname='BEFORE_WITH', opcode=53, arg=None, argval=None, argrepr='', offset=216, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='STORE_FAST', opcode=125, arg=1, argval='dodgy', argrepr='dodgy', offset=218, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=220, starts_line=26, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=230, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=232, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=242, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=244, starts_line=25, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=246, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=248, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=2, argval=2, argrepr='', offset=250, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=260, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=262, starts_line=28, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=272, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=274, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=284, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RETURN_CONST', opcode=121, arg=0, argval=None, argrepr='None', offset=286, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=288, starts_line=25, is_jump_target=False, positions=None), + Instruction(opname='WITH_EXCEPT_START', opcode=49, arg=None, argval=None, argrepr='', offset=290, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=115, arg=1, argval=296, argrepr='to 296', offset=292, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=119, arg=2, argval=2, argrepr='', offset=294, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=296, starts_line=None, is_jump_target=True, positions=None), + Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=298, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=300, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=302, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=140, arg=22, argval=262, argrepr='to 262', offset=304, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=306, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=308, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=310, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=312, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=314, starts_line=22, is_jump_target=False, positions=None), + Instruction(opname='CHECK_EXC_MATCH', opcode=36, arg=None, argval=None, argrepr='', offset=324, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=15, argval=358, argrepr='to 358', offset=326, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=328, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=330, starts_line=23, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=340, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=342, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=352, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=354, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=140, arg=48, argval=262, argrepr='to 262', offset=356, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=119, arg=0, argval=0, argrepr='', offset=358, starts_line=22, is_jump_target=True, positions=None), + Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=360, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=362, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=364, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=366, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=368, starts_line=28, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=378, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=380, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=390, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=119, arg=0, argval=0, argrepr='', offset=392, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=394, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=396, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=398, starts_line=None, is_jump_target=False, positions=None), ] # One last piece of inspect fodder to check the default line number handling diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-09-13-57-35.gh-issue-90997.J-Yhn2.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-09-13-57-35.gh-issue-90997.J-Yhn2.rst new file mode 100644 index 000000000000000..723a4b9fa777d64 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-09-13-57-35.gh-issue-90997.J-Yhn2.rst @@ -0,0 +1,2 @@ +Shrink the number of inline :opcode:`CACHE` entries used by +:opcode:`LOAD_GLOBAL`. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 45d507266743211..dc2a20f22b6038f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1078,7 +1078,7 @@ dummy_func( LOAD_GLOBAL_BUILTIN, }; - inst(LOAD_GLOBAL, (unused/1, unused/1, unused/2, unused/1 -- null if (oparg & 1), v)) { + inst(LOAD_GLOBAL, (unused/1, unused/1, unused/1, unused/1 -- null if (oparg & 1), v)) { #if ENABLE_SPECIALIZATION _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1133,7 +1133,7 @@ dummy_func( null = NULL; } - inst(LOAD_GLOBAL_MODULE, (unused/1, index/1, version/2, unused/1 -- null if (oparg & 1), res)) { + inst(LOAD_GLOBAL_MODULE, (unused/1, index/1, version/1, unused/1 -- null if (oparg & 1), res)) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); PyDictObject *dict = (PyDictObject *)GLOBALS(); @@ -1147,7 +1147,7 @@ dummy_func( null = NULL; } - inst(LOAD_GLOBAL_BUILTIN, (unused/1, index/1, mod_version/2, bltn_version/1 -- null if (oparg & 1), res)) { + inst(LOAD_GLOBAL_BUILTIN, (unused/1, index/1, mod_version/1, bltn_version/1 -- null if (oparg & 1), res)) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 51357cda4c8949c..3839aeecb912a10 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1349,7 +1349,7 @@ TARGET(LOAD_GLOBAL) { PREDICTED(LOAD_GLOBAL); - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 5, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *null = NULL; PyObject *v; #if ENABLE_SPECIALIZATION @@ -1408,7 +1408,7 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = v; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } - next_instr += 5; + next_instr += 4; DISPATCH(); } @@ -1416,7 +1416,7 @@ PyObject *null = NULL; PyObject *res; uint16_t index = read_u16(&next_instr[1].cache); - uint32_t version = read_u32(&next_instr[2].cache); + uint16_t version = read_u16(&next_instr[2].cache); assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); PyDictObject *dict = (PyDictObject *)GLOBALS(); @@ -1432,7 +1432,7 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } - next_instr += 5; + next_instr += 4; DISPATCH(); } @@ -1440,8 +1440,8 @@ PyObject *null = NULL; PyObject *res; uint16_t index = read_u16(&next_instr[1].cache); - uint32_t mod_version = read_u32(&next_instr[2].cache); - uint16_t bltn_version = read_u16(&next_instr[4].cache); + uint16_t mod_version = read_u16(&next_instr[2].cache); + uint16_t bltn_version = read_u16(&next_instr[3].cache); assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); @@ -1460,7 +1460,7 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } - next_instr += 5; + next_instr += 4; DISPATCH(); } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 67cb0088c3b7893..93f3c76c5b82406 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -708,7 +708,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { #endif enum Direction { DIR_NONE, DIR_READ, DIR_WRITE }; -enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC0, INSTR_FMT_IBC000, INSTR_FMT_IBC0000, INSTR_FMT_IBC00000000, INSTR_FMT_IBIB, INSTR_FMT_IX, INSTR_FMT_IXC, INSTR_FMT_IXC000 }; +enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC0, INSTR_FMT_IBC000, INSTR_FMT_IBC00000000, INSTR_FMT_IBIB, INSTR_FMT_IX, INSTR_FMT_IXC, INSTR_FMT_IXC000 }; struct opcode_metadata { enum Direction dir_op1; enum Direction dir_op2; @@ -790,9 +790,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [STORE_GLOBAL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [DELETE_GLOBAL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [LOAD_NAME] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [LOAD_GLOBAL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0000 }, - [LOAD_GLOBAL_MODULE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0000 }, - [LOAD_GLOBAL_BUILTIN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0000 }, + [LOAD_GLOBAL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, + [LOAD_GLOBAL_MODULE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, + [LOAD_GLOBAL_BUILTIN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [DELETE_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [MAKE_CELL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [DELETE_DEREF] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, diff --git a/Python/specialize.c b/Python/specialize.c index 0a7af8991eec83c..719bd5bda329ff3 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1148,8 +1148,12 @@ _Py_Specialize_LoadGlobal( SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS); goto fail; } + if (keys_version != (uint16_t)keys_version) { + SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE); + goto fail; + } cache->index = (uint16_t)index; - write_u32(cache->module_keys_version, keys_version); + cache->module_keys_version = (uint16_t)keys_version; instr->op.code = LOAD_GLOBAL_MODULE; goto success; } @@ -1177,6 +1181,10 @@ _Py_Specialize_LoadGlobal( SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS); goto fail; } + if (globals_version != (uint16_t)globals_version) { + SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE); + goto fail; + } uint32_t builtins_version = _PyDictKeys_GetVersionForCurrentState( interp, builtin_keys); if (builtins_version == 0) { @@ -1188,7 +1196,7 @@ _Py_Specialize_LoadGlobal( goto fail; } cache->index = (uint16_t)index; - write_u32(cache->module_keys_version, globals_version); + cache->module_keys_version = (uint16_t)globals_version; cache->builtin_keys_version = (uint16_t)builtins_version; instr->op.code = LOAD_GLOBAL_BUILTIN; goto success; From 5ffdaf748d98da6065158534720f1996a45a0072 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sat, 11 Mar 2023 01:20:20 +0000 Subject: [PATCH 61/97] gh-102433: Add tests for how classes with properties interact with `isinstance()` checks on `typing.runtime_checkable` protocols (#102449) Co-authored-by: Carl Meyer --- Lib/test/test_typing.py | 88 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 0483ca3aa42f94b..c17be6cd0bbc4a5 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -2530,6 +2530,94 @@ def meth(x): ... with self.assertRaises(TypeError): isinstance(C(), BadPG) + def test_protocols_isinstance_properties_and_descriptors(self): + class C: + @property + def attr(self): + return 42 + + class CustomDescriptor: + def __get__(self, obj, objtype=None): + return 42 + + class D: + attr = CustomDescriptor() + + # Check that properties set on superclasses + # are still found by the isinstance() logic + class E(C): ... + class F(D): ... + + class Empty: ... + + T = TypeVar('T') + + @runtime_checkable + class P(Protocol): + @property + def attr(self): ... + + @runtime_checkable + class P1(Protocol): + attr: int + + @runtime_checkable + class PG(Protocol[T]): + @property + def attr(self): ... + + @runtime_checkable + class PG1(Protocol[T]): + attr: T + + for protocol_class in P, P1, PG, PG1: + for klass in C, D, E, F: + with self.subTest( + klass=klass.__name__, + protocol_class=protocol_class.__name__ + ): + self.assertIsInstance(klass(), protocol_class) + + with self.subTest(klass="Empty", protocol_class=protocol_class.__name__): + self.assertNotIsInstance(Empty(), protocol_class) + + class BadP(Protocol): + @property + def attr(self): ... + + class BadP1(Protocol): + attr: int + + class BadPG(Protocol[T]): + @property + def attr(self): ... + + class BadPG1(Protocol[T]): + attr: T + + for obj in PG[T], PG[C], PG1[T], PG1[C], BadP, BadP1, BadPG, BadPG1: + for klass in C, D, E, F, Empty: + with self.subTest(klass=klass.__name__, obj=obj): + with self.assertRaises(TypeError): + isinstance(klass(), obj) + + def test_protocols_isinstance_not_fooled_by_custom_dir(self): + @runtime_checkable + class HasX(Protocol): + x: int + + class CustomDirWithX: + x = 10 + def __dir__(self): + return [] + + class CustomDirWithoutX: + def __dir__(self): + return ["x"] + + self.assertIsInstance(CustomDirWithX(), HasX) + self.assertNotIsInstance(CustomDirWithoutX(), HasX) + def test_protocols_isinstance_py36(self): class APoint: def __init__(self, x, y, label): From aa0a73d1bc53dcb6348a869df1e775138991e561 Mon Sep 17 00:00:00 2001 From: wangxiang-hz <34048878+wangxiang-hz@users.noreply.github.com> Date: Sat, 11 Mar 2023 19:11:37 +0800 Subject: [PATCH 62/97] gh-102213: Optimize the performance of `__getattr__` (GH-102248) When __getattr__ is defined, python with try to find an attribute using _PyObject_GenericGetAttrWithDict find nothing is reasonable so we don't need an exception, it will hurt performance. --- Include/internal/pycore_object.h | 1 + .../2023-02-26-13-12-55.gh-issue-102213.fTH8X7.rst | 1 + Objects/object.c | 6 ++++++ Objects/typeobject.c | 9 ++++++--- 4 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-26-13-12-55.gh-issue-102213.fTH8X7.rst diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index e15685f174ebcfe..318e6f3371c0c35 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -370,6 +370,7 @@ extern void _PyObject_FreeInstanceAttributes(PyObject *obj); extern int _PyObject_IsInstanceDictEmpty(PyObject *); extern int _PyType_HasSubclasses(PyTypeObject *); extern PyObject* _PyType_GetSubclasses(PyTypeObject *); +extern PyObject* _PyObject_GenericTryGetAttr(PyObject *, PyObject *); // Access macro to the members which are floating "behind" the object static inline PyMemberDef* _PyHeapType_GET_MEMBERS(PyHeapTypeObject *etype) { diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-26-13-12-55.gh-issue-102213.fTH8X7.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-26-13-12-55.gh-issue-102213.fTH8X7.rst new file mode 100644 index 000000000000000..997bef226e713f3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-26-13-12-55.gh-issue-102213.fTH8X7.rst @@ -0,0 +1 @@ +Fix performance loss when accessing an object's attributes with ``__getattr__`` defined. diff --git a/Objects/object.c b/Objects/object.c index 38da4d497a96e72..dff5e2afa16ab84 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1405,6 +1405,12 @@ PyObject_GenericGetAttr(PyObject *obj, PyObject *name) return _PyObject_GenericGetAttrWithDict(obj, name, NULL, 0); } +PyObject * +_PyObject_GenericTryGetAttr(PyObject *obj, PyObject *name) +{ + return _PyObject_GenericGetAttrWithDict(obj, name, NULL, 1); +} + int _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, PyObject *value, PyObject *dict) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index f486b83fd69e648..f0654c239f66358 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8247,14 +8247,17 @@ _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name) (Py_IS_TYPE(getattribute, &PyWrapperDescr_Type) && ((PyWrapperDescrObject *)getattribute)->d_wrapped == (void *)PyObject_GenericGetAttr)) - res = PyObject_GenericGetAttr(self, name); + /* finding nothing is reasonable when __getattr__ is defined */ + res = _PyObject_GenericTryGetAttr(self, name); else { Py_INCREF(getattribute); res = call_attribute(self, getattribute, name); Py_DECREF(getattribute); } - if (res == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { - PyErr_Clear(); + if (res == NULL) { + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } res = call_attribute(self, getattr, name); } Py_DECREF(getattr); From ced13c96a4eb9391a9d27e3e13218f70c579670f Mon Sep 17 00:00:00 2001 From: Thomas Krennwallner Date: Sat, 11 Mar 2023 08:19:40 -0500 Subject: [PATCH 63/97] gh-79940: add introspection API for asynchronous generators to `inspect` module (#11590) --- Doc/library/inspect.rst | 28 ++++- Doc/whatsnew/3.12.rst | 4 + Lib/inspect.py | 50 ++++++++ Lib/test/test_inspect.py | 107 ++++++++++++++++++ ...3-02-26-17-29-57.gh-issue-79940.SAfmAy.rst | 2 + Objects/genobject.c | 10 ++ 6 files changed, 199 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-26-17-29-57.gh-issue-79940.SAfmAy.rst diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 789e9839d22f71d..ccf240193d36a9e 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -1440,8 +1440,8 @@ code execution:: pass -Current State of Generators and Coroutines ------------------------------------------- +Current State of Generators, Coroutines, and Asynchronous Generators +-------------------------------------------------------------------- When implementing coroutine schedulers and for other advanced uses of generators, it is useful to determine whether a generator is currently @@ -1476,6 +1476,22 @@ generator to be determined easily. .. versionadded:: 3.5 +.. function:: getasyncgenstate(agen) + + Get current state of an asynchronous generator object. The function is + intended to be used with asynchronous iterator objects created by + :keyword:`async def` functions which use the :keyword:`yield` statement, + but will accept any asynchronous generator-like object that has + ``ag_running`` and ``ag_frame`` attributes. + + Possible states are: + * AGEN_CREATED: Waiting to start execution. + * AGEN_RUNNING: Currently being executed by the interpreter. + * AGEN_SUSPENDED: Currently suspended at a yield expression. + * AGEN_CLOSED: Execution has completed. + + .. versionadded:: 3.12 + The current internal state of the generator can also be queried. This is mostly useful for testing purposes, to ensure that internal state is being updated as expected: @@ -1507,6 +1523,14 @@ updated as expected: .. versionadded:: 3.5 +.. function:: getasyncgenlocals(agen) + + This function is analogous to :func:`~inspect.getgeneratorlocals`, but + works for asynchronous generator objects created by :keyword:`async def` + functions which use the :keyword:`yield` statement. + + .. versionadded:: 3.12 + .. _inspect-module-co-flags: diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 48b7aab0595ebb2..9f33dbc808ddc02 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -244,6 +244,10 @@ inspect a :term:`coroutine` for use with :func:`iscoroutinefunction`. (Contributed Carlton Gibson in :gh:`99247`.) +* Add :func:`inspect.getasyncgenstate` and :func:`inspect.getasyncgenlocals` + for determining the current state of asynchronous generators. + (Contributed by Thomas Krennwallner in :issue:`35759`.) + pathlib ------- diff --git a/Lib/inspect.py b/Lib/inspect.py index edc23b0ffa9201f..0eceaaf9a24f5d6 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -34,6 +34,10 @@ 'Yury Selivanov ') __all__ = [ + "AGEN_CLOSED", + "AGEN_CREATED", + "AGEN_RUNNING", + "AGEN_SUSPENDED", "ArgInfo", "Arguments", "Attribute", @@ -77,6 +81,8 @@ "getabsfile", "getargs", "getargvalues", + "getasyncgenlocals", + "getasyncgenstate", "getattr_static", "getblock", "getcallargs", @@ -1935,6 +1941,50 @@ def getcoroutinelocals(coroutine): return {} +# ----------------------------------- asynchronous generator introspection + +AGEN_CREATED = 'AGEN_CREATED' +AGEN_RUNNING = 'AGEN_RUNNING' +AGEN_SUSPENDED = 'AGEN_SUSPENDED' +AGEN_CLOSED = 'AGEN_CLOSED' + + +def getasyncgenstate(agen): + """Get current state of an asynchronous generator object. + + Possible states are: + AGEN_CREATED: Waiting to start execution. + AGEN_RUNNING: Currently being executed by the interpreter. + AGEN_SUSPENDED: Currently suspended at a yield expression. + AGEN_CLOSED: Execution has completed. + """ + if agen.ag_running: + return AGEN_RUNNING + if agen.ag_suspended: + return AGEN_SUSPENDED + if agen.ag_frame is None: + return AGEN_CLOSED + return AGEN_CREATED + + +def getasyncgenlocals(agen): + """ + Get the mapping of asynchronous generator local variables to their current + values. + + A dict is returned, with the keys the local variable names and values the + bound values.""" + + if not isasyncgen(agen): + raise TypeError(f"{agen!r} is not a Python async generator") + + frame = getattr(agen, "ag_frame", None) + if frame is not None: + return agen.ag_frame.f_locals + else: + return {} + + ############################################################################### ### Function Signature Object (PEP 362) ############################################################################### diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 02f8378d0413eac..410a2e5b5468f02 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -1,3 +1,4 @@ +import asyncio import builtins import collections import datetime @@ -65,6 +66,10 @@ def revise(filename, *args): git = mod.StupidGit() +def tearDownModule(): + asyncio.set_event_loop_policy(None) + + def signatures_with_lexicographic_keyword_only_parameters(): """ Yields a whole bunch of functions with only keyword-only parameters, @@ -2321,6 +2326,108 @@ async def func(a=None): {'a': None, 'gencoro': gencoro, 'b': 'spam'}) +class TestGetAsyncGenState(unittest.IsolatedAsyncioTestCase): + + def setUp(self): + async def number_asyncgen(): + for number in range(5): + yield number + self.asyncgen = number_asyncgen() + + async def asyncTearDown(self): + await self.asyncgen.aclose() + + def _asyncgenstate(self): + return inspect.getasyncgenstate(self.asyncgen) + + def test_created(self): + self.assertEqual(self._asyncgenstate(), inspect.AGEN_CREATED) + + async def test_suspended(self): + value = await anext(self.asyncgen) + self.assertEqual(self._asyncgenstate(), inspect.AGEN_SUSPENDED) + self.assertEqual(value, 0) + + async def test_closed_after_exhaustion(self): + countdown = 7 + with self.assertRaises(StopAsyncIteration): + while countdown := countdown - 1: + await anext(self.asyncgen) + self.assertEqual(countdown, 1) + self.assertEqual(self._asyncgenstate(), inspect.AGEN_CLOSED) + + async def test_closed_after_immediate_exception(self): + with self.assertRaises(RuntimeError): + await self.asyncgen.athrow(RuntimeError) + self.assertEqual(self._asyncgenstate(), inspect.AGEN_CLOSED) + + async def test_running(self): + async def running_check_asyncgen(): + for number in range(5): + self.assertEqual(self._asyncgenstate(), inspect.AGEN_RUNNING) + yield number + self.assertEqual(self._asyncgenstate(), inspect.AGEN_RUNNING) + self.asyncgen = running_check_asyncgen() + # Running up to the first yield + await anext(self.asyncgen) + self.assertEqual(self._asyncgenstate(), inspect.AGEN_SUSPENDED) + # Running after the first yield + await anext(self.asyncgen) + self.assertEqual(self._asyncgenstate(), inspect.AGEN_SUSPENDED) + + def test_easy_debugging(self): + # repr() and str() of a asyncgen state should contain the state name + names = 'AGEN_CREATED AGEN_RUNNING AGEN_SUSPENDED AGEN_CLOSED'.split() + for name in names: + state = getattr(inspect, name) + self.assertIn(name, repr(state)) + self.assertIn(name, str(state)) + + async def test_getasyncgenlocals(self): + async def each(lst, a=None): + b=(1, 2, 3) + for v in lst: + if v == 3: + c = 12 + yield v + + numbers = each([1, 2, 3]) + self.assertEqual(inspect.getasyncgenlocals(numbers), + {'a': None, 'lst': [1, 2, 3]}) + await anext(numbers) + self.assertEqual(inspect.getasyncgenlocals(numbers), + {'a': None, 'lst': [1, 2, 3], 'v': 1, + 'b': (1, 2, 3)}) + await anext(numbers) + self.assertEqual(inspect.getasyncgenlocals(numbers), + {'a': None, 'lst': [1, 2, 3], 'v': 2, + 'b': (1, 2, 3)}) + await anext(numbers) + self.assertEqual(inspect.getasyncgenlocals(numbers), + {'a': None, 'lst': [1, 2, 3], 'v': 3, + 'b': (1, 2, 3), 'c': 12}) + with self.assertRaises(StopAsyncIteration): + await anext(numbers) + self.assertEqual(inspect.getasyncgenlocals(numbers), {}) + + async def test_getasyncgenlocals_empty(self): + async def yield_one(): + yield 1 + one = yield_one() + self.assertEqual(inspect.getasyncgenlocals(one), {}) + await anext(one) + self.assertEqual(inspect.getasyncgenlocals(one), {}) + with self.assertRaises(StopAsyncIteration): + await anext(one) + self.assertEqual(inspect.getasyncgenlocals(one), {}) + + def test_getasyncgenlocals_error(self): + self.assertRaises(TypeError, inspect.getasyncgenlocals, 1) + self.assertRaises(TypeError, inspect.getasyncgenlocals, lambda x: True) + self.assertRaises(TypeError, inspect.getasyncgenlocals, set) + self.assertRaises(TypeError, inspect.getasyncgenlocals, (2,3)) + + class MySignature(inspect.Signature): # Top-level to make it picklable; # used in test_signature_object_pickle diff --git a/Misc/NEWS.d/next/Library/2023-02-26-17-29-57.gh-issue-79940.SAfmAy.rst b/Misc/NEWS.d/next/Library/2023-02-26-17-29-57.gh-issue-79940.SAfmAy.rst new file mode 100644 index 000000000000000..31b8ead8433279b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-26-17-29-57.gh-issue-79940.SAfmAy.rst @@ -0,0 +1,2 @@ +Add :func:`inspect.getasyncgenstate` and :func:`inspect.getasyncgenlocals`. +Patch by Thomas Krennwallner. diff --git a/Objects/genobject.c b/Objects/genobject.c index 61463774310f884..6316fa9865fe651 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1520,6 +1520,15 @@ ag_getcode(PyGenObject *gen, void *Py_UNUSED(ignored)) return _gen_getcode(gen, "ag_code"); } +static PyObject * +ag_getsuspended(PyAsyncGenObject *ag, void *Py_UNUSED(ignored)) +{ + if (ag->ag_frame_state == FRAME_SUSPENDED) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + static PyGetSetDef async_gen_getsetlist[] = { {"__name__", (getter)gen_get_name, (setter)gen_set_name, PyDoc_STR("name of the async generator")}, @@ -1529,6 +1538,7 @@ static PyGetSetDef async_gen_getsetlist[] = { PyDoc_STR("object being awaited on, or None")}, {"ag_frame", (getter)ag_getframe, NULL, NULL}, {"ag_code", (getter)ag_getcode, NULL, NULL}, + {"ag_suspended", (getter)ag_getsuspended, NULL, NULL}, {NULL} /* Sentinel */ }; From 534660f1680955c7a6a47d5c6bd9649704b74a87 Mon Sep 17 00:00:00 2001 From: Thomas Krennwallner Date: Sat, 11 Mar 2023 13:36:49 -0500 Subject: [PATCH 64/97] gh-79940: skip `TestGetAsyncGenState` on wasm as it requires working sockets (GH-102605) Skip `TestGetAsyncGenState` and restoring of the default event loop policy in `test_inspect` if platform lacks working socket support. Fixes #11590 Automerge-Triggered-By: GH:kumaraditya303 --- Lib/test/test_inspect.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 410a2e5b5468f02..803b259d961f540 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -67,7 +67,8 @@ def revise(filename, *args): def tearDownModule(): - asyncio.set_event_loop_policy(None) + if support.has_socket_support: + asyncio.set_event_loop_policy(None) def signatures_with_lexicographic_keyword_only_parameters(): @@ -2326,6 +2327,7 @@ async def func(a=None): {'a': None, 'gencoro': gencoro, 'b': 'spam'}) +@support.requires_working_socket() class TestGetAsyncGenState(unittest.IsolatedAsyncioTestCase): def setUp(self): From bb396eece44036a71427e7766fbb8e0247373102 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sat, 11 Mar 2023 22:10:52 +0300 Subject: [PATCH 65/97] gh-101821: Test coverage for `ast.main` function (#101822) --- Lib/test/test_ast.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 7c9a57c685df753..6c932e1305e1dd6 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -11,6 +11,7 @@ from textwrap import dedent from test import support +from test.support import os_helper, script_helper from test.support.ast_helper import ASTTestMixin def to_tuple(t): @@ -2564,6 +2565,25 @@ def test_subinterpreter(self): self.assertEqual(res, 0) +class ASTMainTests(unittest.TestCase): + # Tests `ast.main()` function. + + def test_cli_file_input(self): + code = "print(1, 2, 3)" + expected = ast.dump(ast.parse(code), indent=3) + + with os_helper.temp_dir() as tmp_dir: + filename = os.path.join(tmp_dir, "test_module.py") + with open(filename, 'w', encoding='utf-8') as f: + f.write(code) + res, _ = script_helper.run_python_until_end("-m", "ast", filename) + + self.assertEqual(res.err, b"") + self.assertEqual(expected.splitlines(), + res.out.decode("utf8").splitlines()) + self.assertEqual(res.rc, 0) + + def main(): if __name__ != '__main__': return From e6210621bee4ac10e18b4adc11229b8cc1ee788d Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 12 Mar 2023 09:58:02 -0700 Subject: [PATCH 66/97] Fixes duplicated word (#102623) In line 1627, the end of the sentence reads "only that that it may be." but it should read "only that it may be" (or alternatively "only that that may be"). Co-authored-by: Hugo Gabriel Eyherabide --- Doc/library/subprocess.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index ccc431b2d92e072..2b5a82e0107fb6a 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -1622,7 +1622,7 @@ that. It is safe to set these to false on any Python version. They will have no effect on older versions when unsupported. Do not assume the attributes are available to read. Despite their names, a true value does not indicate that the -corresponding function will be used, only that that it may be. +corresponding function will be used, only that it may be. Please file issues any time you have to use these private knobs with a way to reproduce the issue you were seeing. Link to that issue from a comment in your From 6cd7572f859a32a1f4626644c3e8139055df59e3 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 12 Mar 2023 12:48:25 -0500 Subject: [PATCH 67/97] Optimize fmean() weighted average (#102626) --- Lib/statistics.py | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/Lib/statistics.py b/Lib/statistics.py index 07d1fd5ba6e98e9..7d5d750193a5abd 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -136,9 +136,9 @@ from decimal import Decimal from itertools import count, groupby, repeat from bisect import bisect_left, bisect_right -from math import hypot, sqrt, fabs, exp, erf, tau, log, fsum +from math import hypot, sqrt, fabs, exp, erf, tau, log, fsum, sumprod from functools import reduce -from operator import mul, itemgetter +from operator import itemgetter from collections import Counter, namedtuple, defaultdict _SQRT2 = sqrt(2.0) @@ -496,28 +496,26 @@ def fmean(data, weights=None): >>> fmean([3.5, 4.0, 5.25]) 4.25 """ - try: - n = len(data) - except TypeError: - # Handle iterators that do not define __len__(). - n = 0 - def count(iterable): - nonlocal n - for n, x in enumerate(iterable, start=1): - yield x - data = count(data) if weights is None: + try: + n = len(data) + except TypeError: + # Handle iterators that do not define __len__(). + n = 0 + def count(iterable): + nonlocal n + for n, x in enumerate(iterable, start=1): + yield x + data = count(data) total = fsum(data) if not n: raise StatisticsError('fmean requires at least one data point') return total / n - try: - num_weights = len(weights) - except TypeError: + if not isinstance(weights, (list, tuple)): weights = list(weights) - num_weights = len(weights) - num = fsum(map(mul, data, weights)) - if n != num_weights: + try: + num = sumprod(data, weights) + except ValueError: raise StatisticsError('data and weights must be the same length') den = fsum(weights) if not den: From f6ca71a42268dcd890bd1930501b6c7e6d7ce66e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B8=D1=82=D0=B0=D0=BB=D0=B8=D0=B9=20=D0=94=D0=BC?= =?UTF-8?q?=D0=B8=D1=82=D1=80=D0=B8=D0=B5=D0=B2?= Date: Sun, 12 Mar 2023 21:03:59 +0300 Subject: [PATCH 68/97] Fix duplicated words in the docs (GH-102629) With grep utility found some duplicated words Automerge-Triggered-By: GH:AlexWaygood --- Doc/library/base64.rst | 2 +- Doc/reference/compound_stmts.rst | 2 +- Misc/NEWS.d/3.11.0a6.rst | 2 +- Misc/NEWS.d/3.12.0a1.rst | 5 ++--- Misc/NEWS.d/3.12.0a2.rst | 2 +- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index 4ca3768f827c6b5..d5b6af8c1928eff 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -58,7 +58,7 @@ The modern interface provides: This allows an application to e.g. generate URL or filesystem safe Base64 strings. The default is ``None``, for which the standard Base64 alphabet is used. - May assert or raise a a :exc:`ValueError` if the length of *altchars* is not 2. Raises a + May assert or raise a :exc:`ValueError` if the length of *altchars* is not 2. Raises a :exc:`TypeError` if *altchars* is not a :term:`bytes-like object`. diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 52bb0b22b042f61..f0a8936c35bf4a0 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -816,7 +816,7 @@ keyword against a subject. Syntax: If the OR pattern fails, the AS pattern fails. Otherwise, the AS pattern binds the subject to the name on the right of the as keyword and succeeds. -``capture_pattern`` cannot be a a ``_``. +``capture_pattern`` cannot be a ``_``. In simple terms ``P as NAME`` will match with ``P``, and on success it will set ``NAME = ``. diff --git a/Misc/NEWS.d/3.11.0a6.rst b/Misc/NEWS.d/3.11.0a6.rst index 68b80e46690dc90..8621edcfb04bb34 100644 --- a/Misc/NEWS.d/3.11.0a6.rst +++ b/Misc/NEWS.d/3.11.0a6.rst @@ -1055,7 +1055,7 @@ Patch by Victor Stinner. .. section: Build Building Python now requires support for floating point Not-a-Number (NaN): -remove the ``Py_NO_NAN`` macro. Patch by by Victor Stinner. +remove the ``Py_NO_NAN`` macro. Patch by Victor Stinner. .. diff --git a/Misc/NEWS.d/3.12.0a1.rst b/Misc/NEWS.d/3.12.0a1.rst index 2a943fe481af99a..075e8da825a331c 100644 --- a/Misc/NEWS.d/3.12.0a1.rst +++ b/Misc/NEWS.d/3.12.0a1.rst @@ -4559,8 +4559,7 @@ The minimum Sphinx version required to build the documentation is now 3.2. .. section: Documentation Augmented documentation of asyncio.create_task(). Clarified the need to keep -strong references to tasks and added a code snippet detailing how to to -this. +strong references to tasks and added a code snippet detailing how to do this. .. @@ -5066,7 +5065,7 @@ are stubs and not functional. .. nonce: Bsswsc .. section: Build -Add a new ``--with-dsymutil`` configure option to to link debug information +Add a new ``--with-dsymutil`` configure option to link debug information in macOS. Patch by Pablo Galindo. .. diff --git a/Misc/NEWS.d/3.12.0a2.rst b/Misc/NEWS.d/3.12.0a2.rst index 41ad8cd22b5d89d..117be21a3221b60 100644 --- a/Misc/NEWS.d/3.12.0a2.rst +++ b/Misc/NEWS.d/3.12.0a2.rst @@ -546,7 +546,7 @@ Clean up refleak on failed module initialisation in :mod:`_zoneinfo` .. nonce: qc_KHr .. section: Library -Clean up refleaks on failed module initialisation in in :mod:`_pickle` +Clean up refleaks on failed module initialisation in :mod:`_pickle` .. From 5d677c556f03a34d1c2d86e4cc96025870c20c12 Mon Sep 17 00:00:00 2001 From: gaogaotiantian Date: Sun, 12 Mar 2023 16:09:55 -0700 Subject: [PATCH 69/97] GH-101673: Fix pdb bug where local variable changes are lost after longlist (#101674) --- Lib/pdb.py | 13 ++------- Lib/test/test_pdb.py | 29 +++++++++++++++++++ ...-02-09-19-40-41.gh-issue-101673.mX-Ppq.rst | 1 + 3 files changed, 32 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-09-19-40-41.gh-issue-101673.mX-Ppq.rst diff --git a/Lib/pdb.py b/Lib/pdb.py index 78d0ce537f1fc2e..f11fc55536810f8 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -107,15 +107,6 @@ def find_function(funcname, filename): return funcname, filename, lineno return None -def getsourcelines(obj): - lines, lineno = inspect.findsource(obj) - if inspect.isframe(obj) and obj.f_globals is obj.f_locals: - # must be a module frame: do not try to cut a block out of it - return lines, 1 - elif inspect.ismodule(obj): - return lines, 1 - return inspect.getblock(lines[lineno:]), lineno+1 - def lasti2lineno(code, lasti): linestarts = list(dis.findlinestarts(code)) linestarts.reverse() @@ -1357,7 +1348,7 @@ def do_longlist(self, arg): filename = self.curframe.f_code.co_filename breaklist = self.get_file_breaks(filename) try: - lines, lineno = getsourcelines(self.curframe) + lines, lineno = inspect.getsourcelines(self.curframe) except OSError as err: self.error(err) return @@ -1373,7 +1364,7 @@ def do_source(self, arg): except: return try: - lines, lineno = getsourcelines(obj) + lines, lineno = inspect.getsourcelines(obj) except (OSError, TypeError) as err: self.error(err) return diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 48f419e62fbbedc..d91bd0b2f03a0f6 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -1474,6 +1474,35 @@ def test_pdb_issue_gh_94215(): (Pdb) continue """ +def test_pdb_issue_gh_101673(): + """See GH-101673 + + Make sure ll won't revert local variable assignment + + >>> def test_function(): + ... a = 1 + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... '!a = 2', + ... 'll', + ... 'p a', + ... 'continue' + ... ]): + ... test_function() + --Return-- + > (3)test_function()->None + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) !a = 2 + (Pdb) ll + 1 def test_function(): + 2 a = 1 + 3 -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) p a + 2 + (Pdb) continue + """ + @support.requires_subprocess() class PdbTestCase(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2023-02-09-19-40-41.gh-issue-101673.mX-Ppq.rst b/Misc/NEWS.d/next/Library/2023-02-09-19-40-41.gh-issue-101673.mX-Ppq.rst new file mode 100644 index 000000000000000..4e673ba98115686 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-09-19-40-41.gh-issue-101673.mX-Ppq.rst @@ -0,0 +1 @@ +Fix a :mod:`pdb` bug where ``ll`` clears the changes to local variables. From 78e4e6c3d71980d4e6687f07afa6ddfc83e29b04 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 13 Mar 2023 11:24:52 +0200 Subject: [PATCH 70/97] gh-101100: Fix Sphinx warnings in `turtle` module (#102340) Co-authored-by: C.A.M. Gerlach --- Doc/library/turtle.rst | 58 ++++++++++++++++++++++-------------------- Lib/turtle.py | 2 +- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index b7eb569c18a6c4f..05392d04e522635 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -1214,7 +1214,7 @@ Appearance will be displayed stretched according to its stretchfactors: *stretch_wid* is stretchfactor perpendicular to its orientation, *stretch_len* is stretchfactor in direction of its orientation, *outline* determines the width - of the shapes's outline. + of the shape's outline. .. doctest:: :skipif: _tkinter is None @@ -1545,7 +1545,7 @@ below: 1. Create an empty Shape object of type "compound". 2. Add as many components to this object as desired, using the - :meth:`addcomponent` method. + :meth:`~Shape.addcomponent` method. For example: @@ -2125,7 +2125,7 @@ Public classes :param cv: a :class:`tkinter.Canvas` - Provides screen oriented methods like :func:`setbg` etc. that are described + Provides screen oriented methods like :func:`bgcolor` etc. that are described above. .. class:: Screen() @@ -2315,7 +2315,9 @@ of this module or which better fits to your needs, e.g. for use in a classroom, you can prepare a configuration file ``turtle.cfg`` which will be read at import time and modify the configuration according to its settings. -The built in configuration would correspond to the following turtle.cfg:: +The built in configuration would correspond to the following ``turtle.cfg``: + +.. code-block:: ini width = 0.5 height = 0.75 @@ -2340,15 +2342,15 @@ The built in configuration would correspond to the following turtle.cfg:: Short explanation of selected entries: -- The first four lines correspond to the arguments of the :meth:`Screen.setup` +- The first four lines correspond to the arguments of the :func:`Screen.setup ` method. - Line 5 and 6 correspond to the arguments of the method - :meth:`Screen.screensize`. + :func:`Screen.screensize `. - *shape* can be any of the built-in shapes, e.g: arrow, turtle, etc. For more info try ``help(shape)``. -- If you want to use no fillcolor (i.e. make the turtle transparent), you have +- If you want to use no fill color (i.e. make the turtle transparent), you have to write ``fillcolor = ""`` (but all nonempty strings must not have quotes in - the cfg-file). + the cfg file). - If you want to reflect the turtle its state, you have to use ``resizemode = auto``. - If you set e.g. ``language = italian`` the docstringdict @@ -2398,6 +2400,8 @@ The :mod:`turtledemo` package directory contains: The demo scripts are: +.. currentmodule:: turtle + .. tabularcolumns:: |l|L|L| +----------------+------------------------------+-----------------------+ @@ -2469,20 +2473,20 @@ Have fun! Changes since Python 2.6 ======================== -- The methods :meth:`Turtle.tracer`, :meth:`Turtle.window_width` and - :meth:`Turtle.window_height` have been eliminated. +- The methods :func:`Turtle.tracer `, :func:`Turtle.window_width ` and + :func:`Turtle.window_height ` have been eliminated. Methods with these names and functionality are now available only as methods of :class:`Screen`. The functions derived from these remain available. (In fact already in Python 2.6 these methods were merely duplications of the corresponding - :class:`TurtleScreen`/:class:`Screen`-methods.) + :class:`TurtleScreen`/:class:`Screen` methods.) -- The method :meth:`Turtle.fill` has been eliminated. - The behaviour of :meth:`begin_fill` and :meth:`end_fill` - have changed slightly: now every filling-process must be completed with an +- The method :func:`!Turtle.fill` has been eliminated. + The behaviour of :func:`begin_fill` and :func:`end_fill` + have changed slightly: now every filling process must be completed with an ``end_fill()`` call. -- A method :meth:`Turtle.filling` has been added. It returns a boolean +- A method :func:`Turtle.filling ` has been added. It returns a boolean value: ``True`` if a filling process is under way, ``False`` otherwise. This behaviour corresponds to a ``fill()`` call without arguments in Python 2.6. @@ -2490,23 +2494,23 @@ Changes since Python 2.6 Changes since Python 3.0 ======================== -- The methods :meth:`Turtle.shearfactor`, :meth:`Turtle.shapetransform` and - :meth:`Turtle.get_shapepoly` have been added. Thus the full range of +- The :class:`Turtle` methods :func:`shearfactor`, :func:`shapetransform` and + :func:`get_shapepoly` have been added. Thus the full range of regular linear transforms is now available for transforming turtle shapes. - :meth:`Turtle.tiltangle` has been enhanced in functionality: it now can - be used to get or set the tiltangle. :meth:`Turtle.settiltangle` has been + :func:`tiltangle` has been enhanced in functionality: it now can + be used to get or set the tilt angle. :func:`settiltangle` has been deprecated. -- The method :meth:`Screen.onkeypress` has been added as a complement to - :meth:`Screen.onkey` which in fact binds actions to the keyrelease event. - Accordingly the latter has got an alias: :meth:`Screen.onkeyrelease`. +- The :class:`Screen` method :func:`onkeypress` has been added as a complement to + :func:`onkey`. As the latter binds actions to the key release event, + an alias: :func:`onkeyrelease` was also added for it. -- The method :meth:`Screen.mainloop` has been added. So when working only - with Screen and Turtle objects one must not additionally import - :func:`mainloop` anymore. +- The method :func:`Screen.mainloop ` has been added, + so there is no longer a need to use the standalone :func:`mainloop` function + when working with :class:`Screen` and :class:`Turtle` objects. -- Two input methods has been added :meth:`Screen.textinput` and - :meth:`Screen.numinput`. These popup input dialogs and return +- Two input methods have been added: :func:`Screen.textinput ` and + :func:`Screen.numinput `. These pop up input dialogs and return strings and numbers respectively. - Two example scripts :file:`tdemo_nim.py` and :file:`tdemo_round_dance.py` diff --git a/Lib/turtle.py b/Lib/turtle.py index 6abf9f7f65af0d6..1b369327bc8eff0 100644 --- a/Lib/turtle.py +++ b/Lib/turtle.py @@ -954,7 +954,7 @@ def __repr__(self): class TurtleScreen(TurtleScreenBase): - """Provides screen oriented methods like setbg etc. + """Provides screen oriented methods like bgcolor etc. Only relies upon the methods of TurtleScreenBase and NOT upon components of the underlying graphics toolkit - From 233e32f93614255bf5fc7c93cd98af453e58cc98 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 13 Mar 2023 10:34:54 +0000 Subject: [PATCH 71/97] GH-102300: Reuse objects with refcount == 1 in float specialized binary ops. (GH-102301) --- ...-02-27-15-48-31.gh-issue-102300.8o-_Mt.rst | 1 + Python/bytecodes.c | 15 +++----------- Python/ceval_macros.h | 20 +++++++++++++++++++ Python/generated_cases.c.h | 15 +++----------- 4 files changed, 27 insertions(+), 24 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-27-15-48-31.gh-issue-102300.8o-_Mt.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-27-15-48-31.gh-issue-102300.8o-_Mt.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-27-15-48-31.gh-issue-102300.8o-_Mt.rst new file mode 100644 index 000000000000000..4227014582b7e60 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-27-15-48-31.gh-issue-102300.8o-_Mt.rst @@ -0,0 +1 @@ +Reuse operands with refcount of 1 in float specializations of BINARY_OP. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index dc2a20f22b6038f..178519fcb63cede 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -195,10 +195,7 @@ dummy_func( STAT_INC(BINARY_OP, hit); double dprod = ((PyFloatObject *)left)->ob_fval * ((PyFloatObject *)right)->ob_fval; - prod = PyFloat_FromDouble(dprod); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - ERROR_IF(prod == NULL, error); + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dprod, prod); } inst(BINARY_OP_SUBTRACT_INT, (unused/1, left, right -- sub)) { @@ -218,10 +215,7 @@ dummy_func( DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval; - sub = PyFloat_FromDouble(dsub); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - ERROR_IF(sub == NULL, error); + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsub, sub); } inst(BINARY_OP_ADD_UNICODE, (unused/1, left, right -- res)) { @@ -278,10 +272,7 @@ dummy_func( STAT_INC(BINARY_OP, hit); double dsum = ((PyFloatObject *)left)->ob_fval + ((PyFloatObject *)right)->ob_fval; - sum = PyFloat_FromDouble(dsum); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - ERROR_IF(sum == NULL, error); + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsum, sum); } inst(BINARY_OP_ADD_INT, (unused/1, left, right -- sum)) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index ac1fec77daca8a2..98b72ec1b364285 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -350,3 +350,23 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define KWNAMES_LEN() \ (kwnames == NULL ? 0 : ((int)PyTuple_GET_SIZE(kwnames))) + +#define DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dval, result) \ +do { \ + if (Py_REFCNT(left) == 1) { \ + ((PyFloatObject *)left)->ob_fval = (dval); \ + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);\ + result = (left); \ + } \ + else if (Py_REFCNT(right) == 1) {\ + ((PyFloatObject *)right)->ob_fval = (dval); \ + _Py_DECREF_NO_DEALLOC(left); \ + result = (right); \ + }\ + else { \ + result = PyFloat_FromDouble(dval); \ + if ((result) == NULL) goto error; \ + _Py_DECREF_NO_DEALLOC(left); \ + _Py_DECREF_NO_DEALLOC(right); \ + } \ +} while (0) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 3839aeecb912a10..21073cbf978ce0e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -263,10 +263,7 @@ STAT_INC(BINARY_OP, hit); double dprod = ((PyFloatObject *)left)->ob_fval * ((PyFloatObject *)right)->ob_fval; - prod = PyFloat_FromDouble(dprod); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - if (prod == NULL) goto pop_2_error; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dprod, prod); STACK_SHRINK(1); stack_pointer[-1] = prod; next_instr += 1; @@ -300,10 +297,7 @@ DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval; - sub = PyFloat_FromDouble(dsub); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - if (sub == NULL) goto pop_2_error; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsub, sub); STACK_SHRINK(1); stack_pointer[-1] = sub; next_instr += 1; @@ -372,10 +366,7 @@ STAT_INC(BINARY_OP, hit); double dsum = ((PyFloatObject *)left)->ob_fval + ((PyFloatObject *)right)->ob_fval; - sum = PyFloat_FromDouble(dsum); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - if (sum == NULL) goto pop_2_error; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsum, sum); STACK_SHRINK(1); stack_pointer[-1] = sum; next_instr += 1; From 9a8b66b58c74236959a01d579e2c156d9c2e7cb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=AD=E4=B9=9D=E9=BC=8E?= <109224573@qq.com> Date: Mon, 13 Mar 2023 21:15:52 +0800 Subject: [PATCH 72/97] Fix broken link to MSDN (#102355) --- Lib/socket.py | 2 +- Lib/test/support/socket_helper.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/socket.py b/Lib/socket.py index 3a4f94de9cc03a9..321fcda51505c18 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -910,7 +910,7 @@ def create_server(address, *, family=AF_INET, backlog=None, reuse_port=False, # address, effectively preventing this one from accepting # connections. Also, it may set the process in a state where # it'll no longer respond to any signals or graceful kills. - # See: msdn2.microsoft.com/en-us/library/ms740621(VS.85).aspx + # See: https://learn.microsoft.com/windows/win32/winsock/using-so-reuseaddr-and-so-exclusiveaddruse if os.name not in ('nt', 'cygwin') and \ hasattr(_socket, 'SO_REUSEADDR'): try: diff --git a/Lib/test/support/socket_helper.py b/Lib/test/support/socket_helper.py index d2960c9e3334743..78409238db8954d 100644 --- a/Lib/test/support/socket_helper.py +++ b/Lib/test/support/socket_helper.py @@ -63,7 +63,7 @@ def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM): http://bugs.python.org/issue2550 for more info. The following site also has a very thorough description about the implications of both REUSEADDR and EXCLUSIVEADDRUSE on Windows: - http://msdn2.microsoft.com/en-us/library/ms740621(VS.85).aspx) + https://learn.microsoft.com/windows/win32/winsock/using-so-reuseaddr-and-so-exclusiveaddruse XXX: although this approach is a vast improvement on previous attempts to elicit unused ports, it rests heavily on the assumption that the ephemeral From ca01cae1e91446c70fa569d4502c7cbba381e7ff Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Mon, 13 Mar 2023 15:56:24 +0000 Subject: [PATCH 73/97] gh-102192: Replace PyErr_Fetch/Restore etc by more efficient alternatives (#102631) --- Python/ceval.c | 42 +++++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index b422d0ed34ede3d..92235519c993b6a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -13,7 +13,7 @@ #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_moduleobject.h" // PyModuleObject #include "pycore_opcode.h" // EXTRA_CASES -#include "pycore_pyerrors.h" // _PyErr_Fetch(), _PyErr_GetRaisedException() +#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pymem.h" // _PyMem_IsPtrFreed() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_range.h" // _PyRangeIterObject @@ -1783,18 +1783,15 @@ do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause) if (exc == NULL) { /* Reraise */ _PyErr_StackItem *exc_info = _PyErr_GetTopmostException(tstate); - value = exc_info->exc_value; - if (Py_IsNone(value) || value == NULL) { + exc = exc_info->exc_value; + if (Py_IsNone(exc) || exc == NULL) { _PyErr_SetString(tstate, PyExc_RuntimeError, "No active exception to reraise"); return 0; } - assert(PyExceptionInstance_Check(value)); - type = PyExceptionInstance_Class(value); - Py_XINCREF(type); - Py_XINCREF(value); - PyObject *tb = PyException_GetTraceback(value); /* new ref */ - _PyErr_Restore(tstate, type, value, tb); + Py_INCREF(exc); + assert(PyExceptionInstance_Check(exc)); + _PyErr_SetRaisedException(tstate, exc); return 1; } @@ -2035,28 +2032,27 @@ call_exc_trace(Py_tracefunc func, PyObject *self, PyThreadState *tstate, _PyInterpreterFrame *f) { - PyObject *type, *value, *traceback, *orig_traceback, *arg; - int err; - _PyErr_Fetch(tstate, &type, &value, &orig_traceback); - if (value == NULL) { - value = Py_NewRef(Py_None); + PyObject *exc = _PyErr_GetRaisedException(tstate); + assert(exc && PyExceptionInstance_Check(exc)); + PyObject *type = PyExceptionInstance_Class(exc); + PyObject *traceback = PyException_GetTraceback(exc); + if (traceback == NULL) { + traceback = Py_NewRef(Py_None); } - _PyErr_NormalizeException(tstate, &type, &value, &orig_traceback); - traceback = (orig_traceback != NULL) ? orig_traceback : Py_None; - arg = PyTuple_Pack(3, type, value, traceback); + PyObject *arg = PyTuple_Pack(3, type, exc, traceback); + Py_XDECREF(traceback); + if (arg == NULL) { - _PyErr_Restore(tstate, type, value, orig_traceback); + _PyErr_SetRaisedException(tstate, exc); return; } - err = call_trace(func, self, tstate, f, PyTrace_EXCEPTION, arg); + int err = call_trace(func, self, tstate, f, PyTrace_EXCEPTION, arg); Py_DECREF(arg); if (err == 0) { - _PyErr_Restore(tstate, type, value, orig_traceback); + _PyErr_SetRaisedException(tstate, exc); } else { - Py_XDECREF(type); - Py_XDECREF(value); - Py_XDECREF(orig_traceback); + Py_XDECREF(exc); } } From 634cb61909b1392f0eef3e749931777adb54f16e Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Mon, 13 Mar 2023 15:59:20 +0000 Subject: [PATCH 74/97] gh-87092: refactor assemble() to a number of separate functions, which do not need the compiler struct (#102562) --- Lib/test/support/bytecode_helper.py | 2 +- Lib/test/test_peepholer.py | 43 ++- Python/compile.c | 534 +++++++++++++++------------- 3 files changed, 318 insertions(+), 261 deletions(-) diff --git a/Lib/test/support/bytecode_helper.py b/Lib/test/support/bytecode_helper.py index 190fe8723b1fb57..1d9b889c9209865 100644 --- a/Lib/test/support/bytecode_helper.py +++ b/Lib/test/support/bytecode_helper.py @@ -125,7 +125,7 @@ def complete_insts_info(self, insts): assert isinstance(item, tuple) inst = list(reversed(item)) opcode = dis.opmap[inst.pop()] - oparg = inst.pop() if opcode in self.HAS_ARG_OR_TARGET else 0 + oparg = inst.pop() loc = inst + [-1] * (4 - len(inst)) res.append((opcode, oparg, *loc)) return res diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index aea234e38705a80..9ff017da53c2b19 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -995,15 +995,19 @@ def test_conditional_jump_forward_non_const_condition(self): ('LOAD_CONST', 2, 13), lbl, ('LOAD_CONST', 3, 14), + ('RETURN_VALUE', 14), ] - expected = [ + expected_insts = [ ('LOAD_NAME', 1, 11), ('POP_JUMP_IF_TRUE', lbl := self.Label(), 12), - ('LOAD_CONST', 2, 13), + ('LOAD_CONST', 1, 13), lbl, - ('LOAD_CONST', 3, 14) + ('RETURN_CONST', 2, 14), ] - self.cfg_optimization_test(insts, expected, consts=list(range(5))) + self.cfg_optimization_test(insts, + expected_insts, + consts=[0, 1, 2, 3, 4], + expected_consts=[0, 2, 3]) def test_conditional_jump_forward_const_condition(self): # The unreachable branch of the jump is removed, the jump @@ -1015,26 +1019,32 @@ def test_conditional_jump_forward_const_condition(self): ('LOAD_CONST', 2, 13), lbl, ('LOAD_CONST', 3, 14), + ('RETURN_VALUE', 14), ] - expected = [ - ('NOP', None, 11), - ('NOP', None, 12), - ('LOAD_CONST', 3, 14) + expected_insts = [ + ('NOP', 11), + ('NOP', 12), + ('RETURN_CONST', 1, 14), ] - self.cfg_optimization_test(insts, expected, consts=list(range(5))) + self.cfg_optimization_test(insts, + expected_insts, + consts=[0, 1, 2, 3, 4], + expected_consts=[0, 3]) def test_conditional_jump_backward_non_const_condition(self): insts = [ lbl1 := self.Label(), ('LOAD_NAME', 1, 11), ('POP_JUMP_IF_TRUE', lbl1, 12), - ('LOAD_CONST', 2, 13), + ('LOAD_NAME', 2, 13), + ('RETURN_VALUE', 13), ] expected = [ lbl := self.Label(), ('LOAD_NAME', 1, 11), ('POP_JUMP_IF_TRUE', lbl, 12), - ('LOAD_CONST', 2, 13) + ('LOAD_NAME', 2, 13), + ('RETURN_VALUE', 13), ] self.cfg_optimization_test(insts, expected, consts=list(range(5))) @@ -1042,16 +1052,17 @@ def test_conditional_jump_backward_const_condition(self): # The unreachable branch of the jump is removed insts = [ lbl1 := self.Label(), - ('LOAD_CONST', 1, 11), + ('LOAD_CONST', 3, 11), ('POP_JUMP_IF_TRUE', lbl1, 12), ('LOAD_CONST', 2, 13), + ('RETURN_VALUE', 13), ] - expected = [ + expected_insts = [ lbl := self.Label(), - ('NOP', None, 11), - ('JUMP', lbl, 12) + ('NOP', 11), + ('JUMP', lbl, 12), ] - self.cfg_optimization_test(insts, expected, consts=list(range(5))) + self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) if __name__ == "__main__": diff --git a/Python/compile.c b/Python/compile.c index 45c97b4f8ef0e65..29e55b8b30c56b6 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -308,7 +308,7 @@ typedef struct basicblock_ { block, not to be confused with b_next, which is next by control flow. */ struct basicblock_ *b_list; /* The label of this block if it is a jump target, -1 otherwise */ - int b_label; + jump_target_label b_label; /* Exception stack at start of block, used by assembler to create the exception handling table */ ExceptStack *b_exceptstack; /* pointer to an array of instructions, initially NULL */ @@ -501,7 +501,7 @@ instr_sequence_next_inst(instr_sequence *seq) { static jump_target_label instr_sequence_new_label(instr_sequence *seq) { - jump_target_label lbl = {seq->s_next_free_label++}; + jump_target_label lbl = {++seq->s_next_free_label}; return lbl; } @@ -606,6 +606,14 @@ instr_sequence_to_cfg(instr_sequence *seq, cfg_builder *g) { instruction *instr = &seq->s_instrs[i]; RETURN_IF_ERROR(cfg_builder_addop(g, instr->i_opcode, instr->i_oparg, instr->i_loc)); } + int nblocks = 0; + for (basicblock *b = g->g_block_list; b != NULL; b = b->b_list) { + nblocks++; + } + if ((size_t)nblocks > SIZE_MAX / sizeof(basicblock *)) { + PyErr_NoMemory(); + return ERROR; + } return SUCCESS; } @@ -937,6 +945,7 @@ dictbytype(PyObject *src, int scope_type, int flag, Py_ssize_t offset) static bool cfg_builder_check(cfg_builder *g) { + assert(g->g_entryblock->b_iused > 0); for (basicblock *block = g->g_block_list; block != NULL; block = block->b_list) { assert(!_PyMem_IsPtrFreed(block)); if (block->b_instr != NULL) { @@ -1086,7 +1095,7 @@ cfg_builder_new_block(cfg_builder *g) /* Extend the singly linked list of blocks with new block. */ b->b_list = g->g_block_list; g->g_block_list = b; - b->b_label = -1; + b->b_label = NO_LABEL; return b; } @@ -1267,11 +1276,21 @@ basicblock_addop(basicblock *b, int opcode, int oparg, location loc) static bool cfg_builder_current_block_is_terminated(cfg_builder *g) { - if (IS_LABEL(g->g_current_label)) { + struct cfg_instr *last = basicblock_last_instr(g->g_curblock); + if (last && IS_TERMINATOR_OPCODE(last->i_opcode)) { return true; } - struct cfg_instr *last = basicblock_last_instr(g->g_curblock); - return last && IS_TERMINATOR_OPCODE(last->i_opcode); + if (IS_LABEL(g->g_current_label)) { + if (last || IS_LABEL(g->g_curblock->b_label)) { + return true; + } + else { + /* current block is empty, label it */ + g->g_curblock->b_label = g->g_current_label; + g->g_current_label = NO_LABEL; + } + } + return false; } static int @@ -1282,7 +1301,7 @@ cfg_builder_maybe_start_new_block(cfg_builder *g) if (b == NULL) { return ERROR; } - b->b_label = g->g_current_label.id; + b->b_label = g->g_current_label; g->g_current_label = NO_LABEL; cfg_builder_use_next_block(g, b); } @@ -1432,50 +1451,49 @@ merge_consts_recursive(PyObject *const_cache, PyObject *o) } static Py_ssize_t -compiler_add_const(struct compiler *c, PyObject *o) +compiler_add_const(PyObject *const_cache, struct compiler_unit *u, PyObject *o) { - PyObject *key = merge_consts_recursive(c->c_const_cache, o); + assert(PyDict_CheckExact(const_cache)); + PyObject *key = merge_consts_recursive(const_cache, o); if (key == NULL) { return ERROR; } - Py_ssize_t arg = dict_add_o(c->u->u_consts, key); + Py_ssize_t arg = dict_add_o(u->u_consts, key); Py_DECREF(key); return arg; } static int -compiler_addop_load_const(struct compiler *c, location loc, PyObject *o) +compiler_addop_load_const(PyObject *const_cache, struct compiler_unit *u, location loc, PyObject *o) { - Py_ssize_t arg = compiler_add_const(c, o); + Py_ssize_t arg = compiler_add_const(const_cache, u, o); if (arg < 0) { return ERROR; } - return codegen_addop_i(INSTR_SEQUENCE(c), LOAD_CONST, arg, loc); + return codegen_addop_i(&u->u_instr_sequence, LOAD_CONST, arg, loc); } static int -compiler_addop_o(struct compiler *c, location loc, +compiler_addop_o(struct compiler_unit *u, location loc, int opcode, PyObject *dict, PyObject *o) { Py_ssize_t arg = dict_add_o(dict, o); if (arg < 0) { return ERROR; } - return codegen_addop_i(INSTR_SEQUENCE(c), opcode, arg, loc); + return codegen_addop_i(&u->u_instr_sequence, opcode, arg, loc); } static int -compiler_addop_name(struct compiler *c, location loc, +compiler_addop_name(struct compiler_unit *u, location loc, int opcode, PyObject *dict, PyObject *o) { - Py_ssize_t arg; - - PyObject *mangled = _Py_Mangle(c->u->u_private, o); + PyObject *mangled = _Py_Mangle(u->u_private, o); if (!mangled) { return ERROR; } - arg = dict_add_o(dict, mangled); + Py_ssize_t arg = dict_add_o(dict, mangled); Py_DECREF(mangled); if (arg < 0) { return ERROR; @@ -1488,7 +1506,7 @@ compiler_addop_name(struct compiler *c, location loc, arg <<= 1; arg |= 1; } - return codegen_addop_i(INSTR_SEQUENCE(c), opcode, arg, loc); + return codegen_addop_i(&u->u_instr_sequence, opcode, arg, loc); } /* Add an opcode with an integer argument */ @@ -1509,7 +1527,7 @@ codegen_addop_i(instr_sequence *seq, int opcode, Py_ssize_t oparg, location loc) static int codegen_addop_j(instr_sequence *seq, location loc, - int opcode, jump_target_label target) + int opcode, jump_target_label target) { assert(IS_LABEL(target)); assert(IS_JUMP_OPCODE(opcode) || IS_BLOCK_PUSH_OPCODE(opcode)); @@ -1527,7 +1545,7 @@ codegen_addop_j(instr_sequence *seq, location loc, } #define ADDOP_LOAD_CONST(C, LOC, O) \ - RETURN_IF_ERROR(compiler_addop_load_const((C), (LOC), (O))) + RETURN_IF_ERROR(compiler_addop_load_const((C)->c_const_cache, (C)->u, (LOC), (O))) /* Same as ADDOP_LOAD_CONST, but steals a reference. */ #define ADDOP_LOAD_CONST_NEW(C, LOC, O) { \ @@ -1535,7 +1553,7 @@ codegen_addop_j(instr_sequence *seq, location loc, if (__new_const == NULL) { \ return ERROR; \ } \ - if (compiler_addop_load_const((C), (LOC), __new_const) < 0) { \ + if (compiler_addop_load_const((C)->c_const_cache, (C)->u, (LOC), __new_const) < 0) { \ Py_DECREF(__new_const); \ return ERROR; \ } \ @@ -1544,7 +1562,7 @@ codegen_addop_j(instr_sequence *seq, location loc, #define ADDOP_N(C, LOC, OP, O, TYPE) { \ assert(!HAS_CONST(OP)); /* use ADDOP_LOAD_CONST_NEW */ \ - if (compiler_addop_o((C), (LOC), (OP), (C)->u->u_ ## TYPE, (O)) < 0) { \ + if (compiler_addop_o((C)->u, (LOC), (OP), (C)->u->u_ ## TYPE, (O)) < 0) { \ Py_DECREF((O)); \ return ERROR; \ } \ @@ -1552,7 +1570,7 @@ codegen_addop_j(instr_sequence *seq, location loc, } #define ADDOP_NAME(C, LOC, OP, O, TYPE) \ - RETURN_IF_ERROR(compiler_addop_name((C), (LOC), (OP), (C)->u->u_ ## TYPE, (O))) + RETURN_IF_ERROR(compiler_addop_name((C)->u, (LOC), (OP), (C)->u->u_ ## TYPE, (O))) #define ADDOP_I(C, LOC, OP, O) \ RETURN_IF_ERROR(codegen_addop_i(INSTR_SEQUENCE(C), (OP), (O), (LOC))) @@ -2556,7 +2574,7 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) if (c->c_optimize < 2) { docstring = _PyAST_GetDocString(body); } - if (compiler_add_const(c, docstring ? docstring : Py_None) < 0) { + if (compiler_add_const(c->c_const_cache, c->u, docstring ? docstring : Py_None) < 0) { compiler_exit_scope(c); return ERROR; } @@ -2924,7 +2942,7 @@ compiler_lambda(struct compiler *c, expr_ty e) /* Make None the first constant, so the lambda can't have a docstring. */ - RETURN_IF_ERROR(compiler_add_const(c, Py_None)); + RETURN_IF_ERROR(compiler_add_const(c->c_const_cache, c->u, Py_None)); c->u->u_argcount = asdl_seq_LEN(args->args); c->u->u_posonlyargcount = asdl_seq_LEN(args->posonlyargs); @@ -4915,7 +4933,7 @@ compiler_call_simple_kw_helper(struct compiler *c, location loc, keyword_ty kw = asdl_seq_GET(keywords, i); PyTuple_SET_ITEM(names, i, Py_NewRef(kw->arg)); } - Py_ssize_t arg = compiler_add_const(c, names); + Py_ssize_t arg = compiler_add_const(c->c_const_cache, c->u, names); if (arg < 0) { return ERROR; } @@ -7372,7 +7390,7 @@ push_cold_blocks_to_end(cfg_builder *g, int code_flags) { if (explicit_jump == NULL) { return ERROR; } - basicblock_addop(explicit_jump, JUMP, b->b_next->b_label, NO_LOCATION); + basicblock_addop(explicit_jump, JUMP, b->b_next->b_label.id, NO_LOCATION); explicit_jump->b_cold = 1; explicit_jump->b_next = b->b_next; b->b_next = explicit_jump; @@ -7676,13 +7694,33 @@ assemble_emit_location(struct assembler* a, location loc, int isize) return write_location_info_entry(a, loc, isize); } -/* assemble_emit() +static int +assemble_location_info(struct assembler *a, basicblock *entryblock, int firstlineno) +{ + a->a_lineno = firstlineno; + location loc = NO_LOCATION; + int size = 0; + for (basicblock *b = entryblock; b != NULL; b = b->b_next) { + for (int j = 0; j < b->b_iused; j++) { + if (!same_location(loc, b->b_instr[j].i_loc)) { + RETURN_IF_ERROR(assemble_emit_location(a, loc, size)); + loc = b->b_instr[j].i_loc; + size = 0; + } + size += instr_size(&b->b_instr[j]); + } + } + RETURN_IF_ERROR(assemble_emit_location(a, loc, size)); + return SUCCESS; +} + +/* assemble_emit_instr() Extend the bytecode with a new instruction. Update lnotab if necessary. */ static int -assemble_emit(struct assembler *a, struct cfg_instr *i) +assemble_emit_instr(struct assembler *a, struct cfg_instr *i) { Py_ssize_t len = PyBytes_GET_SIZE(a->a_bytecode); _Py_CODEUNIT *code; @@ -7700,6 +7738,35 @@ assemble_emit(struct assembler *a, struct cfg_instr *i) return SUCCESS; } +static int merge_const_one(PyObject *const_cache, PyObject **obj); + +static int +assemble_emit(struct assembler *a, basicblock *entryblock, int first_lineno, + PyObject *const_cache) +{ + RETURN_IF_ERROR(assemble_init(a, first_lineno)); + + for (basicblock *b = entryblock; b != NULL; b = b->b_next) { + for (int j = 0; j < b->b_iused; j++) { + RETURN_IF_ERROR(assemble_emit_instr(a, &b->b_instr[j])); + } + } + + RETURN_IF_ERROR(assemble_location_info(a, entryblock, a->a_lineno)); + + RETURN_IF_ERROR(assemble_exception_table(a, entryblock)); + + RETURN_IF_ERROR(_PyBytes_Resize(&a->a_except_table, a->a_except_table_off)); + RETURN_IF_ERROR(merge_const_one(const_cache, &a->a_except_table)); + + RETURN_IF_ERROR(_PyBytes_Resize(&a->a_linetable, a->a_location_off)); + RETURN_IF_ERROR(merge_const_one(const_cache, &a->a_linetable)); + + RETURN_IF_ERROR(_PyBytes_Resize(&a->a_bytecode, a->a_offset * sizeof(_Py_CODEUNIT))); + RETURN_IF_ERROR(merge_const_one(const_cache, &a->a_bytecode)); + return SUCCESS; +} + static int normalize_jumps_in_block(cfg_builder *g, basicblock *b) { struct cfg_instr *last = basicblock_last_instr(b); @@ -7760,7 +7827,7 @@ normalize_jumps_in_block(cfg_builder *g, basicblock *b) { if (backwards_jump == NULL) { return ERROR; } - basicblock_addop(backwards_jump, JUMP, target->b_label, NO_LOCATION); + basicblock_addop(backwards_jump, JUMP, target->b_label.id, NO_LOCATION); backwards_jump->b_instr[0].i_target = target; last->i_opcode = reversed_opcode; last->i_target = b->b_next; @@ -7963,9 +8030,9 @@ fast_scan_many_locals(basicblock *entryblock, int nlocals) static int add_checks_for_loads_of_uninitialized_variables(basicblock *entryblock, - struct compiler *c) + int nlocals, + int nparams) { - int nlocals = (int)PyDict_GET_SIZE(c->u->u_varnames); if (nlocals == 0) { return SUCCESS; } @@ -7986,7 +8053,6 @@ add_checks_for_loads_of_uninitialized_variables(basicblock *entryblock, // First origin of being uninitialized: // The non-parameter locals in the entry block. - int nparams = (int)PyList_GET_SIZE(c->u->u_ste->ste_varnames); uint64_t start_mask = 0; for (int i = nparams; i < nlocals; i++) { start_mask |= (uint64_t)1 << i; @@ -8092,7 +8158,7 @@ compute_code_flags(struct compiler *c) static int merge_const_one(PyObject *const_cache, PyObject **obj) { - PyDict_CheckExact(const_cache); + assert(PyDict_CheckExact(const_cache)); PyObject *key = _PyCode_ConstantKey(*obj); if (key == NULL) { return ERROR; @@ -8122,29 +8188,29 @@ extern void _Py_set_localsplus_info(int, PyObject *, unsigned char, PyObject *, PyObject *); static void -compute_localsplus_info(struct compiler *c, int nlocalsplus, +compute_localsplus_info(struct compiler_unit *u, int nlocalsplus, PyObject *names, PyObject *kinds) { PyObject *k, *v; Py_ssize_t pos = 0; - while (PyDict_Next(c->u->u_varnames, &pos, &k, &v)) { + while (PyDict_Next(u->u_varnames, &pos, &k, &v)) { int offset = (int)PyLong_AS_LONG(v); assert(offset >= 0); assert(offset < nlocalsplus); // For now we do not distinguish arg kinds. _PyLocals_Kind kind = CO_FAST_LOCAL; - if (PyDict_GetItem(c->u->u_cellvars, k) != NULL) { + if (PyDict_GetItem(u->u_cellvars, k) != NULL) { kind |= CO_FAST_CELL; } _Py_set_localsplus_info(offset, k, kind, names, kinds); } - int nlocals = (int)PyDict_GET_SIZE(c->u->u_varnames); + int nlocals = (int)PyDict_GET_SIZE(u->u_varnames); // This counter mirrors the fix done in fix_cell_offsets(). int numdropped = 0; pos = 0; - while (PyDict_Next(c->u->u_cellvars, &pos, &k, &v)) { - if (PyDict_GetItem(c->u->u_varnames, k) != NULL) { + while (PyDict_Next(u->u_cellvars, &pos, &k, &v)) { + if (PyDict_GetItem(u->u_varnames, k) != NULL) { // Skip cells that are already covered by locals. numdropped += 1; continue; @@ -8157,7 +8223,7 @@ compute_localsplus_info(struct compiler *c, int nlocalsplus, } pos = 0; - while (PyDict_Next(c->u->u_freevars, &pos, &k, &v)) { + while (PyDict_Next(u->u_freevars, &pos, &k, &v)) { int offset = (int)PyLong_AS_LONG(v); assert(offset >= 0); offset += nlocals - numdropped; @@ -8167,19 +8233,20 @@ compute_localsplus_info(struct compiler *c, int nlocalsplus, } static PyCodeObject * -makecode(struct compiler *c, struct assembler *a, PyObject *constslist, - int maxdepth, int nlocalsplus, int code_flags) +makecode(struct compiler_unit *u, struct assembler *a, PyObject *const_cache, + PyObject *constslist, int maxdepth, int nlocalsplus, int code_flags, + PyObject *filename) { PyCodeObject *co = NULL; PyObject *names = NULL; PyObject *consts = NULL; PyObject *localsplusnames = NULL; PyObject *localspluskinds = NULL; - names = dict_keys_inorder(c->u->u_names, 0); + names = dict_keys_inorder(u->u_names, 0); if (!names) { goto error; } - if (merge_const_one(c->c_const_cache, &names) < 0) { + if (merge_const_one(const_cache, &names) < 0) { goto error; } @@ -8187,17 +8254,17 @@ makecode(struct compiler *c, struct assembler *a, PyObject *constslist, if (consts == NULL) { goto error; } - if (merge_const_one(c->c_const_cache, &consts) < 0) { + if (merge_const_one(const_cache, &consts) < 0) { goto error; } - assert(c->u->u_posonlyargcount < INT_MAX); - assert(c->u->u_argcount < INT_MAX); - assert(c->u->u_kwonlyargcount < INT_MAX); - int posonlyargcount = (int)c->u->u_posonlyargcount; - int posorkwargcount = (int)c->u->u_argcount; + assert(u->u_posonlyargcount < INT_MAX); + assert(u->u_argcount < INT_MAX); + assert(u->u_kwonlyargcount < INT_MAX); + int posonlyargcount = (int)u->u_posonlyargcount; + int posorkwargcount = (int)u->u_argcount; assert(INT_MAX - posonlyargcount - posorkwargcount > 0); - int kwonlyargcount = (int)c->u->u_kwonlyargcount; + int kwonlyargcount = (int)u->u_kwonlyargcount; localsplusnames = PyTuple_New(nlocalsplus); if (localsplusnames == NULL) { @@ -8207,16 +8274,16 @@ makecode(struct compiler *c, struct assembler *a, PyObject *constslist, if (localspluskinds == NULL) { goto error; } - compute_localsplus_info(c, nlocalsplus, localsplusnames, localspluskinds); + compute_localsplus_info(u, nlocalsplus, localsplusnames, localspluskinds); struct _PyCodeConstructor con = { - .filename = c->c_filename, - .name = c->u->u_name, - .qualname = c->u->u_qualname ? c->u->u_qualname : c->u->u_name, + .filename = filename, + .name = u->u_name, + .qualname = u->u_qualname ? u->u_qualname : u->u_name, .flags = code_flags, .code = a->a_bytecode, - .firstlineno = c->u->u_firstlineno, + .firstlineno = u->u_firstlineno, .linetable = a->a_linetable, .consts = consts, @@ -8238,7 +8305,7 @@ makecode(struct compiler *c, struct assembler *a, PyObject *constslist, goto error; } - if (merge_const_one(c->c_const_cache, &localsplusnames) < 0) { + if (merge_const_one(const_cache, &localsplusnames) < 0) { goto error; } con.localsplusnames = localsplusnames; @@ -8289,7 +8356,7 @@ dump_basicblock(const basicblock *b) { const char *b_return = basicblock_returns(b) ? "return " : ""; fprintf(stderr, "%d: [EH=%d CLD=%d WRM=%d NO_FT=%d %p] used: %d, depth: %d, offset: %d %s\n", - b->b_label, b->b_except_handler, b->b_cold, b->b_warm, BB_NO_FALLTHROUGH(b), b, b->b_iused, + b->b_label.id, b->b_except_handler, b->b_cold, b->b_warm, BB_NO_FALLTHROUGH(b), b, b->b_iused, b->b_startdepth, b->b_offset, b_return); if (b->b_instr) { int i; @@ -8316,11 +8383,11 @@ static int duplicate_exits_without_lineno(cfg_builder *g); static int * -build_cellfixedoffsets(struct compiler *c) +build_cellfixedoffsets(struct compiler_unit *u) { - int nlocals = (int)PyDict_GET_SIZE(c->u->u_varnames); - int ncellvars = (int)PyDict_GET_SIZE(c->u->u_cellvars); - int nfreevars = (int)PyDict_GET_SIZE(c->u->u_freevars); + int nlocals = (int)PyDict_GET_SIZE(u->u_varnames); + int ncellvars = (int)PyDict_GET_SIZE(u->u_cellvars); + int nfreevars = (int)PyDict_GET_SIZE(u->u_freevars); int noffsets = ncellvars + nfreevars; int *fixed = PyMem_New(int, noffsets); @@ -8334,8 +8401,8 @@ build_cellfixedoffsets(struct compiler *c) PyObject *varname, *cellindex; Py_ssize_t pos = 0; - while (PyDict_Next(c->u->u_cellvars, &pos, &varname, &cellindex)) { - PyObject *varindex = PyDict_GetItem(c->u->u_varnames, varname); + while (PyDict_Next(u->u_cellvars, &pos, &varname, &cellindex)) { + PyObject *varindex = PyDict_GetItem(u->u_varnames, varname); if (varindex != NULL) { assert(PyLong_AS_LONG(cellindex) < INT_MAX); assert(PyLong_AS_LONG(varindex) < INT_MAX); @@ -8349,17 +8416,17 @@ build_cellfixedoffsets(struct compiler *c) } static int -insert_prefix_instructions(struct compiler *c, basicblock *entryblock, +insert_prefix_instructions(struct compiler_unit *u, basicblock *entryblock, int *fixed, int nfreevars, int code_flags) { - assert(c->u->u_firstlineno > 0); + assert(u->u_firstlineno > 0); /* Add the generator prefix instructions. */ if (code_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) { struct cfg_instr make_gen = { .i_opcode = RETURN_GENERATOR, .i_oparg = 0, - .i_loc = LOCATION(c->u->u_firstlineno, c->u->u_firstlineno, -1, -1), + .i_loc = LOCATION(u->u_firstlineno, u->u_firstlineno, -1, -1), .i_target = NULL, }; RETURN_IF_ERROR(insert_instruction(entryblock, 0, &make_gen)); @@ -8373,12 +8440,12 @@ insert_prefix_instructions(struct compiler *c, basicblock *entryblock, } /* Set up cells for any variable that escapes, to be put in a closure. */ - const int ncellvars = (int)PyDict_GET_SIZE(c->u->u_cellvars); + const int ncellvars = (int)PyDict_GET_SIZE(u->u_cellvars); if (ncellvars) { - // c->u->u_cellvars has the cells out of order so we sort them + // u->u_cellvars has the cells out of order so we sort them // before adding the MAKE_CELL instructions. Note that we // adjust for arg cells, which come first. - const int nvars = ncellvars + (int)PyDict_GET_SIZE(c->u->u_varnames); + const int nvars = ncellvars + (int)PyDict_GET_SIZE(u->u_varnames); int *sorted = PyMem_RawCalloc(nvars, sizeof(int)); if (sorted == NULL) { PyErr_NoMemory(); @@ -8447,11 +8514,11 @@ guarantee_lineno_for_exits(basicblock *entryblock, int firstlineno) { } static int -fix_cell_offsets(struct compiler *c, basicblock *entryblock, int *fixedmap) +fix_cell_offsets(struct compiler_unit *u, basicblock *entryblock, int *fixedmap) { - int nlocals = (int)PyDict_GET_SIZE(c->u->u_varnames); - int ncellvars = (int)PyDict_GET_SIZE(c->u->u_cellvars); - int nfreevars = (int)PyDict_GET_SIZE(c->u->u_freevars); + int nlocals = (int)PyDict_GET_SIZE(u->u_varnames); + int ncellvars = (int)PyDict_GET_SIZE(u->u_cellvars); + int nfreevars = (int)PyDict_GET_SIZE(u->u_freevars); int noffsets = ncellvars + nfreevars; // First deal with duplicates (arg cells). @@ -8491,8 +8558,6 @@ fix_cell_offsets(struct compiler *c, basicblock *entryblock, int *fixedmap) return numdropped; } -static void -propagate_line_numbers(basicblock *entryblock); #ifndef NDEBUG @@ -8589,30 +8654,30 @@ remove_redundant_jumps(cfg_builder *g) { } static int -prepare_localsplus(struct compiler* c, cfg_builder *g, int code_flags) +prepare_localsplus(struct compiler_unit* u, cfg_builder *g, int code_flags) { - assert(PyDict_GET_SIZE(c->u->u_varnames) < INT_MAX); - assert(PyDict_GET_SIZE(c->u->u_cellvars) < INT_MAX); - assert(PyDict_GET_SIZE(c->u->u_freevars) < INT_MAX); - int nlocals = (int)PyDict_GET_SIZE(c->u->u_varnames); - int ncellvars = (int)PyDict_GET_SIZE(c->u->u_cellvars); - int nfreevars = (int)PyDict_GET_SIZE(c->u->u_freevars); + assert(PyDict_GET_SIZE(u->u_varnames) < INT_MAX); + assert(PyDict_GET_SIZE(u->u_cellvars) < INT_MAX); + assert(PyDict_GET_SIZE(u->u_freevars) < INT_MAX); + int nlocals = (int)PyDict_GET_SIZE(u->u_varnames); + int ncellvars = (int)PyDict_GET_SIZE(u->u_cellvars); + int nfreevars = (int)PyDict_GET_SIZE(u->u_freevars); assert(INT_MAX - nlocals - ncellvars > 0); assert(INT_MAX - nlocals - ncellvars - nfreevars > 0); int nlocalsplus = nlocals + ncellvars + nfreevars; - int* cellfixedoffsets = build_cellfixedoffsets(c); + int* cellfixedoffsets = build_cellfixedoffsets(u); if (cellfixedoffsets == NULL) { return ERROR; } // This must be called before fix_cell_offsets(). - if (insert_prefix_instructions(c, g->g_entryblock, cellfixedoffsets, nfreevars, code_flags)) { + if (insert_prefix_instructions(u, g->g_entryblock, cellfixedoffsets, nfreevars, code_flags)) { PyMem_Free(cellfixedoffsets); return ERROR; } - int numdropped = fix_cell_offsets(c, g->g_entryblock, cellfixedoffsets); + int numdropped = fix_cell_offsets(u, g->g_entryblock, cellfixedoffsets); PyMem_Free(cellfixedoffsets); // At this point we're done with it. cellfixedoffsets = NULL; if (numdropped < 0) { @@ -8636,176 +8701,127 @@ add_return_at_end(struct compiler *c, int addNone) return SUCCESS; } -static PyCodeObject * -assemble(struct compiler *c, int addNone) -{ - PyCodeObject *co = NULL; - PyObject *consts = NULL; - cfg_builder g_; - cfg_builder *g = &g_; - struct assembler a; - memset(&a, 0, sizeof(struct assembler)); - - int code_flags = compute_code_flags(c); - if (code_flags < 0) { - return NULL; - } - - if (add_return_at_end(c, addNone) < 0) { - return NULL; - } - - /** Preprocessing **/ - if (instr_sequence_to_cfg(INSTR_SEQUENCE(c), g) < 0) { - goto error; - } - assert(cfg_builder_check(g)); - - int nblocks = 0; - for (basicblock *b = g->g_block_list; b != NULL; b = b->b_list) { - nblocks++; - } - if ((size_t)nblocks > SIZE_MAX / sizeof(basicblock *)) { - PyErr_NoMemory(); - goto error; - } +static void propagate_line_numbers(basicblock *entryblock); +static int +resolve_line_numbers(struct compiler_unit *u, cfg_builder *g) +{ /* Set firstlineno if it wasn't explicitly set. */ - if (!c->u->u_firstlineno) { + if (!u->u_firstlineno) { if (g->g_entryblock->b_instr && g->g_entryblock->b_instr->i_loc.lineno) { - c->u->u_firstlineno = g->g_entryblock->b_instr->i_loc.lineno; + u->u_firstlineno = g->g_entryblock->b_instr->i_loc.lineno; } else { - c->u->u_firstlineno = 1; + u->u_firstlineno = 1; } } + RETURN_IF_ERROR(duplicate_exits_without_lineno(g)); + propagate_line_numbers(g->g_entryblock); + guarantee_lineno_for_exits(g->g_entryblock, u->u_firstlineno); + return SUCCESS; +} +static int +optimize_code_unit(cfg_builder *g, PyObject *consts, PyObject *const_cache, + int code_flags, int nlocals, int nparams) +{ + assert(cfg_builder_check(g)); + /** Preprocessing **/ /* Map labels to targets and mark exception handlers */ - if (translate_jump_labels_to_targets(g->g_entryblock) < 0) { - goto error; - } - if (mark_except_handlers(g->g_entryblock) < 0) { - goto error; - } - if (label_exception_targets(g->g_entryblock)) { - goto error; - } + RETURN_IF_ERROR(translate_jump_labels_to_targets(g->g_entryblock)); + RETURN_IF_ERROR(mark_except_handlers(g->g_entryblock)); + RETURN_IF_ERROR(label_exception_targets(g->g_entryblock)); /** Optimization **/ - consts = consts_dict_keys_inorder(c->u->u_consts); + RETURN_IF_ERROR(optimize_cfg(g, consts, const_cache)); + RETURN_IF_ERROR(remove_unused_consts(g->g_entryblock, consts)); + RETURN_IF_ERROR( + add_checks_for_loads_of_uninitialized_variables( + g->g_entryblock, nlocals, nparams)); + + RETURN_IF_ERROR(push_cold_blocks_to_end(g, code_flags)); + return SUCCESS; +} + +static PyCodeObject * +assemble_code_unit(struct compiler_unit *u, PyObject *const_cache, + int code_flags, PyObject *filename) +{ + PyCodeObject *co = NULL; + PyObject *consts = consts_dict_keys_inorder(u->u_consts); if (consts == NULL) { goto error; } - if (optimize_cfg(g, consts, c->c_const_cache)) { - goto error; - } - if (remove_unused_consts(g->g_entryblock, consts) < 0) { + cfg_builder g; + if (instr_sequence_to_cfg(&u->u_instr_sequence, &g) < 0) { goto error; } - if (add_checks_for_loads_of_uninitialized_variables(g->g_entryblock, c) < 0) { + int nparams = (int)PyList_GET_SIZE(u->u_ste->ste_varnames); + int nlocals = (int)PyDict_GET_SIZE(u->u_varnames); + if (optimize_code_unit(&g, consts, const_cache, code_flags, nlocals, nparams) < 0) { goto error; } - /** line numbers (TODO: move this before optimization stage) */ - if (duplicate_exits_without_lineno(g) < 0) { - goto error; - } - propagate_line_numbers(g->g_entryblock); - guarantee_lineno_for_exits(g->g_entryblock, c->u->u_firstlineno); + /** Assembly **/ - if (push_cold_blocks_to_end(g, code_flags) < 0) { + if (resolve_line_numbers(u, &g) < 0) { goto error; } - /** Assembly **/ - - int nlocalsplus = prepare_localsplus(c, g, code_flags); + int nlocalsplus = prepare_localsplus(u, &g, code_flags); if (nlocalsplus < 0) { goto error; } - int maxdepth = stackdepth(g->g_entryblock, code_flags); + int maxdepth = stackdepth(g.g_entryblock, code_flags); if (maxdepth < 0) { goto error; } /* TO DO -- For 3.12, make sure that `maxdepth <= MAX_ALLOWED_STACK_USE` */ - convert_exception_handlers_to_nops(g->g_entryblock); + convert_exception_handlers_to_nops(g.g_entryblock); /* Order of basic blocks must have been determined by now */ - if (normalize_jumps(g) < 0) { + if (normalize_jumps(&g) < 0) { goto error; } - assert(no_redundant_jumps(g)); - assert(opcode_metadata_is_sane(g)); + assert(no_redundant_jumps(&g)); + assert(opcode_metadata_is_sane(&g)); /* Can't modify the bytecode after computing jump offsets. */ - assemble_jump_offsets(g->g_entryblock); + assemble_jump_offsets(g.g_entryblock); - /* Create assembler */ - if (assemble_init(&a, c->u->u_firstlineno) < 0) { - goto error; - } - - /* Emit code. */ - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - for (int j = 0; j < b->b_iused; j++) { - if (assemble_emit(&a, &b->b_instr[j]) < 0) { - goto error; - } - } + struct assembler a; + int res = assemble_emit(&a, g.g_entryblock, u->u_firstlineno, const_cache); + if (res == SUCCESS) { + co = makecode(u, &a, const_cache, consts, maxdepth, nlocalsplus, + code_flags, filename); } + assemble_free(&a); - /* Emit location info */ - a.a_lineno = c->u->u_firstlineno; - location loc = NO_LOCATION; - int size = 0; - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - for (int j = 0; j < b->b_iused; j++) { - if (!same_location(loc, b->b_instr[j].i_loc)) { - if (assemble_emit_location(&a, loc, size)) { - goto error; - } - loc = b->b_instr[j].i_loc; - size = 0; - } - size += instr_size(&b->b_instr[j]); - } - } - if (assemble_emit_location(&a, loc, size)) { - goto error; - } + error: + Py_XDECREF(consts); + cfg_builder_fini(&g); + return co; +} - if (assemble_exception_table(&a, g->g_entryblock) < 0) { - goto error; - } - if (_PyBytes_Resize(&a.a_except_table, a.a_except_table_off) < 0) { - goto error; - } - if (merge_const_one(c->c_const_cache, &a.a_except_table) < 0) { - goto error; - } +static PyCodeObject * +assemble(struct compiler *c, int addNone) +{ + struct compiler_unit *u = c->u; + PyObject *const_cache = c->c_const_cache; + PyObject *filename = c->c_filename; - if (_PyBytes_Resize(&a.a_linetable, a.a_location_off) < 0) { - goto error; - } - if (merge_const_one(c->c_const_cache, &a.a_linetable) < 0) { - goto error; + int code_flags = compute_code_flags(c); + if (code_flags < 0) { + return NULL; } - if (_PyBytes_Resize(&a.a_bytecode, a.a_offset * sizeof(_Py_CODEUNIT)) < 0) { - goto error; - } - if (merge_const_one(c->c_const_cache, &a.a_bytecode) < 0) { - goto error; + if (add_return_at_end(c, addNone) < 0) { + return NULL; } - co = makecode(c, &a, consts, maxdepth, nlocalsplus, code_flags); - error: - Py_XDECREF(consts); - cfg_builder_fini(g); - assemble_free(&a); - return co; + return assemble_code_unit(u, const_cache, code_flags, filename); } static PyObject* @@ -9529,8 +9545,8 @@ translate_jump_labels_to_targets(basicblock *entryblock) { int max_label = -1; for (basicblock *b = entryblock; b != NULL; b = b->b_next) { - if (b->b_label > max_label) { - max_label = b->b_label; + if (b->b_label.id > max_label) { + max_label = b->b_label.id; } } size_t mapsize = sizeof(basicblock *) * (max_label + 1); @@ -9541,8 +9557,8 @@ translate_jump_labels_to_targets(basicblock *entryblock) } memset(label2block, 0, mapsize); for (basicblock *b = entryblock; b != NULL; b = b->b_next) { - if (b->b_label >= 0) { - label2block[b->b_label] = b; + if (b->b_label.id >= 0) { + label2block[b->b_label.id] = b; } } for (basicblock *b = entryblock; b != NULL; b = b->b_next) { @@ -9554,7 +9570,7 @@ translate_jump_labels_to_targets(basicblock *entryblock) assert(lbl >= 0 && lbl <= max_label); instr->i_target = label2block[lbl]; assert(instr->i_target != NULL); - assert(instr->i_target->b_label == lbl); + assert(instr->i_target->b_label.id == lbl); } } } @@ -9828,9 +9844,15 @@ instructions_to_cfg(PyObject *instructions, cfg_builder *g) if (PyErr_Occurred()) { goto error; } - int oparg = PyLong_AsLong(PyTuple_GET_ITEM(item, 1)); - if (PyErr_Occurred()) { - goto error; + int oparg; + if (HAS_ARG(opcode)) { + oparg = PyLong_AsLong(PyTuple_GET_ITEM(item, 1)); + if (PyErr_Occurred()) { + goto error; + } + } + else { + oparg = 0; } location loc; loc.lineno = PyLong_AsLong(PyTuple_GET_ITEM(item, 2)); @@ -9851,7 +9873,10 @@ instructions_to_cfg(PyObject *instructions, cfg_builder *g) } RETURN_IF_ERROR(cfg_builder_addop(g, opcode, oparg, loc)); } - + struct cfg_instr *last = basicblock_last_instr(g->g_curblock); + if (last && !IS_TERMINATOR_OPCODE(last->i_opcode)) { + RETURN_IF_ERROR(cfg_builder_addop(g, RETURN_VALUE, 0, NO_LOCATION)); + } PyMem_Free(is_target); return SUCCESS; error: @@ -9859,6 +9884,38 @@ instructions_to_cfg(PyObject *instructions, cfg_builder *g) return ERROR; } +static PyObject * +instr_sequence_to_instructions(instr_sequence *seq) { + PyObject *instructions = PyList_New(0); + if (instructions == NULL) { + return NULL; + } + for (int i = 0; i < seq->s_used; i++) { + instruction *instr = &seq->s_instrs[i]; + location loc = instr->i_loc; + int arg = HAS_TARGET(instr->i_opcode) ? + seq->s_labelmap[instr->i_oparg] : instr->i_oparg; + + PyObject *inst_tuple = Py_BuildValue( + "(iiiiii)", instr->i_opcode, arg, + loc.lineno, loc.end_lineno, + loc.col_offset, loc.end_col_offset); + if (inst_tuple == NULL) { + goto error; + } + + int res = PyList_Append(instructions, inst_tuple); + Py_DECREF(inst_tuple); + if (res != 0) { + goto error; + } + } + return instructions; +error: + Py_XDECREF(instructions); + return NULL; +} + static PyObject * cfg_to_instructions(cfg_builder *g) { @@ -9868,7 +9925,7 @@ cfg_to_instructions(cfg_builder *g) } int lbl = 0; for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - b->b_label = lbl; + b->b_label = (jump_target_label){lbl}; lbl += b->b_iused; } for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { @@ -9876,7 +9933,7 @@ cfg_to_instructions(cfg_builder *g) struct cfg_instr *instr = &b->b_instr[i]; location loc = instr->i_loc; int arg = HAS_TARGET(instr->i_opcode) ? - instr->i_target->b_label : instr->i_oparg; + instr->i_target->b_label.id : instr->i_oparg; PyObject *inst_tuple = Py_BuildValue( "(iiiiii)", instr->i_opcode, arg, @@ -9932,19 +9989,10 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, goto finally; } - cfg_builder g; - if (instr_sequence_to_cfg(INSTR_SEQUENCE(c), &g) < 0) { - goto finally; - } - if (translate_jump_labels_to_targets(g.g_entryblock) < 0) { - goto finally; - } - - res = cfg_to_instructions(&g); + res = instr_sequence_to_instructions(INSTR_SEQUENCE(c)); finally: compiler_exit_scope(c); - cfg_builder_fini(&g); compiler_free(c); _PyArena_Free(arena); return res; @@ -9954,7 +10002,11 @@ PyObject * _PyCompile_OptimizeCfg(PyObject *instructions, PyObject *consts) { PyObject *res = NULL; - PyObject *const_cache = NULL; + PyObject *const_cache = PyDict_New(); + if (const_cache == NULL) { + return NULL; + } + cfg_builder g; memset(&g, 0, sizeof(cfg_builder)); if (cfg_builder_init(&g) < 0) { @@ -9963,19 +10015,13 @@ _PyCompile_OptimizeCfg(PyObject *instructions, PyObject *consts) if (instructions_to_cfg(instructions, &g) < 0) { goto error; } - const_cache = PyDict_New(); - if (const_cache == NULL) { - goto error; - } - if (translate_jump_labels_to_targets(g.g_entryblock) < 0) { - goto error; - } - if (optimize_cfg(&g, consts, const_cache) < 0) { + int code_flags = 0, nlocals = 0, nparams = 0; + if (optimize_code_unit(&g, consts, const_cache, code_flags, nlocals, nparams) < 0) { goto error; } res = cfg_to_instructions(&g); error: - Py_XDECREF(const_cache); + Py_DECREF(const_cache); cfg_builder_fini(&g); return res; } From 2d370da5702f07d044f0e612df68172852b89f42 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 13 Mar 2023 18:35:37 +0000 Subject: [PATCH 75/97] GH-100987: Don't cache references to the names and consts array in `_PyEval_EvalFrameDefault`. (#102640) * Rename local variables, names and consts, from the interpeter loop. Will allow non-code objects in frames for better introspection of C builtins and extensions. * Remove unused dummy variables. --- Python/bytecodes.c | 44 +++++++++++++++++------------------- Python/ceval.c | 7 ------ Python/generated_cases.c.h | 46 +++++++++++++++++++------------------- 3 files changed, 44 insertions(+), 53 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 178519fcb63cede..a715bfd3289ec94 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -70,8 +70,6 @@ dummy_func( unsigned int oparg, _Py_atomic_int * const eval_breaker, _PyCFrame cframe, - PyObject *names, - PyObject *consts, _Py_CODEUNIT *next_instr, PyObject **stack_pointer, PyObject *kwnames, @@ -115,7 +113,7 @@ dummy_func( } inst(LOAD_CONST, (-- value)) { - value = GETITEM(consts, oparg); + value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); } @@ -554,7 +552,7 @@ dummy_func( } inst(RETURN_CONST, (--)) { - PyObject *retval = GETITEM(consts, oparg); + PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(retval); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -844,7 +842,7 @@ dummy_func( } inst(STORE_NAME, (v -- )) { - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; if (ns == NULL) { @@ -862,7 +860,7 @@ dummy_func( } inst(DELETE_NAME, (--)) { - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; if (ns == NULL) { @@ -956,7 +954,7 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { assert(cframe.use_tracing == 0); - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); next_instr--; _Py_Specialize_StoreAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); @@ -967,7 +965,7 @@ dummy_func( #else (void)counter; // Unused. #endif /* ENABLE_SPECIALIZATION */ - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, v); Py_DECREF(v); Py_DECREF(owner); @@ -975,21 +973,21 @@ dummy_func( } inst(DELETE_ATTR, (owner --)) { - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, (PyObject *)NULL); Py_DECREF(owner); ERROR_IF(err, error); } inst(STORE_GLOBAL, (v --)) { - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); Py_DECREF(v); ERROR_IF(err, error); } inst(DELETE_GLOBAL, (--)) { - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err; err = PyDict_DelItem(GLOBALS(), name); // Can't use ERROR_IF here. @@ -1003,7 +1001,7 @@ dummy_func( } inst(LOAD_NAME, ( -- v)) { - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *locals = LOCALS(); if (locals == NULL) { _PyErr_Format(tstate, PyExc_SystemError, @@ -1074,7 +1072,7 @@ dummy_func( _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); - PyObject *name = GETITEM(names, oparg>>1); + PyObject *name = GETITEM(frame->f_code->co_names, oparg>>1); next_instr--; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); DISPATCH_SAME_OPARG(); @@ -1082,7 +1080,7 @@ dummy_func( STAT_INC(LOAD_GLOBAL, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ - PyObject *name = GETITEM(names, oparg>>1); + PyObject *name = GETITEM(frame->f_code->co_names, oparg>>1); if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { @@ -1436,7 +1434,7 @@ dummy_func( _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); - PyObject *name = GETITEM(names, oparg>>1); + PyObject *name = GETITEM(frame->f_code->co_names, oparg>>1); next_instr--; _Py_Specialize_LoadAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); @@ -1444,7 +1442,7 @@ dummy_func( STAT_INC(LOAD_ATTR, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ - PyObject *name = GETITEM(names, oparg >> 1); + PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 1); if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ PyObject* meth = NULL; @@ -1525,7 +1523,7 @@ dummy_func( PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); DEOPT_IF(dict == NULL, LOAD_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(names, oparg>>1); + PyObject *name = GETITEM(frame->f_code->co_names, oparg>>1); uint16_t hint = index; DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); if (DK_IS_UNICODE(dict->ma_keys)) { @@ -1616,7 +1614,7 @@ dummy_func( DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(names, oparg >> 1); + PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 1); Py_INCREF(f); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); // Manipulate stack directly because we exit with DISPATCH_INLINED(). @@ -1661,7 +1659,7 @@ dummy_func( PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); DEOPT_IF(dict == NULL, STORE_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); PyObject *old_value; uint64_t new_version; @@ -1855,14 +1853,14 @@ dummy_func( } inst(IMPORT_NAME, (level, fromlist -- res)) { - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_name(tstate, frame, name, fromlist, level); DECREF_INPUTS(); ERROR_IF(res == NULL, error); } inst(IMPORT_FROM, (from -- from, res)) { - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_from(tstate, from, name); ERROR_IF(res == NULL, error); } @@ -2360,8 +2358,8 @@ dummy_func( inst(KW_NAMES, (--)) { assert(kwnames == NULL); - assert(oparg < PyTuple_GET_SIZE(consts)); - kwnames = GETITEM(consts, oparg); + assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); + kwnames = GETITEM(frame->f_code->co_consts, oparg); } // Cache layout: counter/1, func_version/2, min_args/1 diff --git a/Python/ceval.c b/Python/ceval.c index 92235519c993b6a..7d60cf987e9c476 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -773,18 +773,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int /* Local "register" variables. * These are cached values from the frame and code object. */ - PyObject *names; - PyObject *consts; _Py_CODEUNIT *next_instr; PyObject **stack_pointer; /* Sets the above local variables from the frame */ #define SET_LOCALS_FROM_FRAME() \ - { \ - PyCodeObject *co = frame->f_code; \ - names = co->co_names; \ - consts = co->co_consts; \ - } \ assert(_PyInterpreterFrame_LASTI(frame) >= -1); \ /* Jump back to the last instruction executed... */ \ next_instr = frame->prev_instr + 1; \ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 21073cbf978ce0e..631b7844f9ed88c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -50,7 +50,7 @@ TARGET(LOAD_CONST) { PREDICTED(LOAD_CONST); PyObject *value; - value = GETITEM(consts, oparg); + value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); STACK_GROW(1); stack_pointer[-1] = value; @@ -101,7 +101,7 @@ oparg = (next_instr++)->op.arg; { PyObject *value; - value = GETITEM(consts, oparg); + value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); _tmp_1 = value; } @@ -150,7 +150,7 @@ PyObject *_tmp_2; { PyObject *value; - value = GETITEM(consts, oparg); + value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); _tmp_2 = value; } @@ -744,7 +744,7 @@ } TARGET(RETURN_CONST) { - PyObject *retval = GETITEM(consts, oparg); + PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(retval); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1083,7 +1083,7 @@ TARGET(STORE_NAME) { PyObject *v = stack_pointer[-1]; - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; if (ns == NULL) { @@ -1103,7 +1103,7 @@ } TARGET(DELETE_NAME) { - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; if (ns == NULL) { @@ -1217,7 +1217,7 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { assert(cframe.use_tracing == 0); - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); next_instr--; _Py_Specialize_StoreAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); @@ -1228,7 +1228,7 @@ #else (void)counter; // Unused. #endif /* ENABLE_SPECIALIZATION */ - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, v); Py_DECREF(v); Py_DECREF(owner); @@ -1240,7 +1240,7 @@ TARGET(DELETE_ATTR) { PyObject *owner = stack_pointer[-1]; - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, (PyObject *)NULL); Py_DECREF(owner); if (err) goto pop_1_error; @@ -1250,7 +1250,7 @@ TARGET(STORE_GLOBAL) { PyObject *v = stack_pointer[-1]; - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); Py_DECREF(v); if (err) goto pop_1_error; @@ -1259,7 +1259,7 @@ } TARGET(DELETE_GLOBAL) { - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err; err = PyDict_DelItem(GLOBALS(), name); // Can't use ERROR_IF here. @@ -1275,7 +1275,7 @@ TARGET(LOAD_NAME) { PyObject *v; - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *locals = LOCALS(); if (locals == NULL) { _PyErr_Format(tstate, PyExc_SystemError, @@ -1347,7 +1347,7 @@ _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); - PyObject *name = GETITEM(names, oparg>>1); + PyObject *name = GETITEM(frame->f_code->co_names, oparg>>1); next_instr--; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); DISPATCH_SAME_OPARG(); @@ -1355,7 +1355,7 @@ STAT_INC(LOAD_GLOBAL, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ - PyObject *name = GETITEM(names, oparg>>1); + PyObject *name = GETITEM(frame->f_code->co_names, oparg>>1); if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { @@ -1797,7 +1797,7 @@ _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); - PyObject *name = GETITEM(names, oparg>>1); + PyObject *name = GETITEM(frame->f_code->co_names, oparg>>1); next_instr--; _Py_Specialize_LoadAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); @@ -1805,7 +1805,7 @@ STAT_INC(LOAD_ATTR, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ - PyObject *name = GETITEM(names, oparg >> 1); + PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 1); if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ PyObject* meth = NULL; @@ -1916,7 +1916,7 @@ PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); DEOPT_IF(dict == NULL, LOAD_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(names, oparg>>1); + PyObject *name = GETITEM(frame->f_code->co_names, oparg>>1); uint16_t hint = index; DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); if (DK_IS_UNICODE(dict->ma_keys)) { @@ -2040,7 +2040,7 @@ DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(names, oparg >> 1); + PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 1); Py_INCREF(f); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); // Manipulate stack directly because we exit with DISPATCH_INLINED(). @@ -2096,7 +2096,7 @@ PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); DEOPT_IF(dict == NULL, STORE_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); PyObject *old_value; uint64_t new_version; @@ -2349,7 +2349,7 @@ PyObject *fromlist = stack_pointer[-1]; PyObject *level = stack_pointer[-2]; PyObject *res; - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_name(tstate, frame, name, fromlist, level); Py_DECREF(level); Py_DECREF(fromlist); @@ -2362,7 +2362,7 @@ TARGET(IMPORT_FROM) { PyObject *from = stack_pointer[-1]; PyObject *res; - PyObject *name = GETITEM(names, oparg); + PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; STACK_GROW(1); @@ -2997,8 +2997,8 @@ TARGET(KW_NAMES) { assert(kwnames == NULL); - assert(oparg < PyTuple_GET_SIZE(consts)); - kwnames = GETITEM(consts, oparg); + assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); + kwnames = GETITEM(frame->f_code->co_consts, oparg); DISPATCH(); } From 85ba8a3e03707092800cbf2a29d95e0b495e3cb7 Mon Sep 17 00:00:00 2001 From: chgnrdv <52372310+chgnrdv@users.noreply.github.com> Date: Mon, 13 Mar 2023 22:25:17 +0300 Subject: [PATCH 76/97] gh-102650: Remove duplicate include directives from multiple source files (#102651) Remove duplicate include directives from multiple source files --- Mac/Tools/pythonw.c | 1 - Modules/_hashopenssl.c | 3 +-- Modules/arraymodule.c | 1 - Modules/signalmodule.c | 4 +--- Programs/_testembed.c | 1 - 5 files changed, 2 insertions(+), 8 deletions(-) diff --git a/Mac/Tools/pythonw.c b/Mac/Tools/pythonw.c index 78813e818e7dace..9dfb77f6ff41c3f 100644 --- a/Mac/Tools/pythonw.c +++ b/Mac/Tools/pythonw.c @@ -27,7 +27,6 @@ #include #include #include -#include #include diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 82398547f9b3720..ee8c588020118cc 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -32,12 +32,11 @@ /* EVP is the preferred interface to hashing in OpenSSL */ #include #include -#include +#include // FIPS_mode() /* We use the object interface to discover what hashes OpenSSL supports. */ #include #include -#include // FIPS_mode() #ifndef OPENSSL_THREADS # error "OPENSSL_THREADS is not defined, Python requires thread-safe OpenSSL" diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 114c69a033593c8..798a76292579666 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -13,7 +13,6 @@ #include "pycore_bytesobject.h" // _PyBytes_Repeat #include "structmember.h" // PyMemberDef #include // offsetof() -#include /*[clinic input] module array diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 0e472e1ee4f9ddc..fdd1450050fa1b9 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -13,7 +13,7 @@ #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pyerrors.h" // _PyErr_SetString() #include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_signal.h" +#include "pycore_signal.h" // Py_NSIG #ifndef MS_WINDOWS # include "posixmodule.h" @@ -28,8 +28,6 @@ # endif #endif -#include "pycore_signal.h" // Py_NSIG - #ifdef HAVE_SIGNAL_H # include #endif diff --git a/Programs/_testembed.c b/Programs/_testembed.c index a6ce3f7b200550c..00717114b40286a 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -9,7 +9,6 @@ #include "pycore_initconfig.h" // _PyConfig_InitCompatConfig() #include "pycore_runtime.h" // _PyRuntime #include "pycore_import.h" // _PyImport_FrozenBootstrap -#include #include #include #include // putenv() From 71e37d907905b0504c5bb7b25681adeea2157492 Mon Sep 17 00:00:00 2001 From: T Date: Tue, 14 Mar 2023 04:46:35 +0800 Subject: [PATCH 77/97] gh-98169 dataclasses.astuple support DefaultDict (#98170) Co-authored-by: Pieter Eendebak --- Lib/dataclasses.py | 25 ++++++++++++------- Lib/test/test_dataclasses.py | 21 +++++++++++++--- ...2-10-10-19-14-51.gh-issue-98169.DBWIxL.rst | 2 ++ 3 files changed, 35 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-10-10-19-14-51.gh-issue-98169.DBWIxL.rst diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index f4617b1dbdac669..7c3285cf440a397 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1321,15 +1321,14 @@ def _asdict_inner(obj, dict_factory): # generator (which is not true for namedtuples, handled # above). return type(obj)(_asdict_inner(v, dict_factory) for v in obj) - elif isinstance(obj, dict) and hasattr(type(obj), 'default_factory'): - # obj is a defaultdict, which has a different constructor from - # dict as it requires the default_factory as its first arg. - # https://bugs.python.org/issue35540 - result = type(obj)(getattr(obj, 'default_factory')) - for k, v in obj.items(): - result[_asdict_inner(k, dict_factory)] = _asdict_inner(v, dict_factory) - return result elif isinstance(obj, dict): + if hasattr(type(obj), 'default_factory'): + # obj is a defaultdict, which has a different constructor from + # dict as it requires the default_factory as its first arg. + result = type(obj)(getattr(obj, 'default_factory')) + for k, v in obj.items(): + result[_asdict_inner(k, dict_factory)] = _asdict_inner(v, dict_factory) + return result return type(obj)((_asdict_inner(k, dict_factory), _asdict_inner(v, dict_factory)) for k, v in obj.items()) @@ -1382,7 +1381,15 @@ def _astuple_inner(obj, tuple_factory): # above). return type(obj)(_astuple_inner(v, tuple_factory) for v in obj) elif isinstance(obj, dict): - return type(obj)((_astuple_inner(k, tuple_factory), _astuple_inner(v, tuple_factory)) + obj_type = type(obj) + if hasattr(obj_type, 'default_factory'): + # obj is a defaultdict, which has a different constructor from + # dict as it requires the default_factory as its first arg. + result = obj_type(getattr(obj, 'default_factory')) + for k, v in obj.items(): + result[_astuple_inner(k, tuple_factory)] = _astuple_inner(v, tuple_factory) + return result + return obj_type((_astuple_inner(k, tuple_factory), _astuple_inner(v, tuple_factory)) for k, v in obj.items()) else: return copy.deepcopy(obj) diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 76bed0c33146734..46d4e0fedad2f27 100644 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -1706,19 +1706,17 @@ class C: def test_helper_asdict_defaultdict(self): # Ensure asdict() does not throw exceptions when a # defaultdict is a member of a dataclass - @dataclass class C: mp: DefaultDict[str, List] - dd = defaultdict(list) dd["x"].append(12) c = C(mp=dd) d = asdict(c) - assert d == {"mp": {"x": [12]}} - assert d["mp"] is not c.mp # make sure defaultdict is copied + self.assertEqual(d, {"mp": {"x": [12]}}) + self.assertTrue(d["mp"] is not c.mp) # make sure defaultdict is copied def test_helper_astuple(self): # Basic tests for astuple(), it should return a new tuple. @@ -1847,6 +1845,21 @@ class C: t = astuple(c, tuple_factory=list) self.assertEqual(t, ['outer', T(1, ['inner', T(11, 12, 13)], 2)]) + def test_helper_astuple_defaultdict(self): + # Ensure astuple() does not throw exceptions when a + # defaultdict is a member of a dataclass + @dataclass + class C: + mp: DefaultDict[str, List] + + dd = defaultdict(list) + dd["x"].append(12) + c = C(mp=dd) + t = astuple(c) + + self.assertEqual(t, ({"x": [12]},)) + self.assertTrue(t[0] is not dd) # make sure defaultdict is copied + def test_dynamic_class_creation(self): cls_dict = {'__annotations__': {'x': int, 'y': int}, } diff --git a/Misc/NEWS.d/next/Library/2022-10-10-19-14-51.gh-issue-98169.DBWIxL.rst b/Misc/NEWS.d/next/Library/2022-10-10-19-14-51.gh-issue-98169.DBWIxL.rst new file mode 100644 index 000000000000000..24c3aeecc83f182 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-10-19-14-51.gh-issue-98169.DBWIxL.rst @@ -0,0 +1,2 @@ +Fix :func:`dataclasses.astuple` crash when :class:`collections.defaultdict` +is present in the attributes. From d97757f793ea53dda3cc6882b4a92d3e921b17c9 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Tue, 14 Mar 2023 00:42:05 +0300 Subject: [PATCH 78/97] gh-102069: Fix `__weakref__` descriptor generation for custom dataclasses (#102075) --- Lib/dataclasses.py | 3 +++ Lib/test/test_dataclasses.py | 15 +++++++++++---- ...2023-02-20-16-47-56.gh-issue-102069.FS7f1j.rst | 1 + 3 files changed, 15 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-20-16-47-56.gh-issue-102069.FS7f1j.rst diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 7c3285cf440a397..82b08fc017884f5 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1189,6 +1189,9 @@ def _add_slots(cls, is_frozen, weakref_slot): # Remove __dict__ itself. cls_dict.pop('__dict__', None) + # Clear existing `__weakref__` descriptor, it belongs to a previous type: + cls_dict.pop('__weakref__', None) # gh-102069 + # And finally create the class. qualname = getattr(cls, '__qualname__', None) cls = type(cls)(cls.__name__, cls.__bases__, cls_dict) diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 46d4e0fedad2f27..46f33043c270714 100644 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -3175,6 +3175,8 @@ class A: with self.assertRaisesRegex(TypeError, "cannot create weak reference"): weakref.ref(a) + with self.assertRaises(AttributeError): + a.__weakref__ def test_slots_weakref(self): @dataclass(slots=True, weakref_slot=True) @@ -3183,7 +3185,9 @@ class A: self.assertIn("__weakref__", A.__slots__) a = A(1) - weakref.ref(a) + a_ref = weakref.ref(a) + + self.assertIs(a.__weakref__, a_ref) def test_slots_weakref_base_str(self): class Base: @@ -3249,7 +3253,8 @@ class A(Base): self.assertIn("__weakref__", Base.__slots__) self.assertNotIn("__weakref__", A.__slots__) a = A(1) - weakref.ref(a) + a_ref = weakref.ref(a) + self.assertIs(a.__weakref__, a_ref) def test_weakref_slot_subclass_no_weakref_slot(self): @dataclass(slots=True, weakref_slot=True) @@ -3265,7 +3270,8 @@ class A(Base): self.assertIn("__weakref__", Base.__slots__) self.assertNotIn("__weakref__", A.__slots__) a = A(1) - weakref.ref(a) + a_ref = weakref.ref(a) + self.assertIs(a.__weakref__, a_ref) def test_weakref_slot_normal_base_weakref_slot(self): class Base: @@ -3280,7 +3286,8 @@ class A(Base): self.assertIn("__weakref__", Base.__slots__) self.assertNotIn("__weakref__", A.__slots__) a = A(1) - weakref.ref(a) + a_ref = weakref.ref(a) + self.assertIs(a.__weakref__, a_ref) class TestDescriptors(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2023-02-20-16-47-56.gh-issue-102069.FS7f1j.rst b/Misc/NEWS.d/next/Library/2023-02-20-16-47-56.gh-issue-102069.FS7f1j.rst new file mode 100644 index 000000000000000..04c87e515cca939 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-20-16-47-56.gh-issue-102069.FS7f1j.rst @@ -0,0 +1 @@ +Fix ``__weakref__`` descriptor generation for custom dataclasses. From 74885a08dbc7cc768d15711752957096d6a5a350 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 13 Mar 2023 15:50:16 -0600 Subject: [PATCH 79/97] gh-101524: Fix the ChannelID tp_name (gh-102655) https://github.com/python/cpython/issues/101524 --- Modules/_xxinterpchannelsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index a0cd4a2363fb53a..fead12c963da26b 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -1806,7 +1806,7 @@ static PyType_Slot ChannelIDType_slots[] = { }; static PyType_Spec ChannelIDType_spec = { - .name = "_xxsubinterpreters.ChannelID", + .name = MODULE_NAME ".ChannelID", .basicsize = sizeof(channelid), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE), From 959ea2f9e967042954878f7af5c85053238212a7 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 13 Mar 2023 16:01:44 -0600 Subject: [PATCH 80/97] gh-101659: Avoid Allocation for Shared Exceptions in the _xxsubinterpreters Module (gh-102659) https://github.com/python/cpython/issues/101659 --- Modules/_xxsubinterpretersmodule.c | 123 ++++++++++++----------------- 1 file changed, 49 insertions(+), 74 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 79dbe3474ba9e81..76fb87fa3a34e1f 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -15,14 +15,14 @@ #define MODULE_NAME "_xxsubinterpreters" -static char * +static const char * _copy_raw_string(PyObject *strobj) { const char *str = PyUnicode_AsUTF8(strobj); if (str == NULL) { return NULL; } - char *copied = PyMem_Malloc(strlen(str)+1); + char *copied = PyMem_RawMalloc(strlen(str)+1); if (copied == NULL) { PyErr_NoMemory(); return NULL; @@ -128,7 +128,7 @@ clear_module_state(module_state *state) /* data-sharing-specific code ***********************************************/ struct _sharednsitem { - char *name; + const char *name; _PyCrossInterpreterData data; }; @@ -152,7 +152,7 @@ static void _sharednsitem_clear(struct _sharednsitem *item) { if (item->name != NULL) { - PyMem_Free(item->name); + PyMem_RawFree((void *)item->name); item->name = NULL; } (void)_release_xid_data(&item->data, 1); @@ -258,96 +258,74 @@ _sharedns_apply(_sharedns *shared, PyObject *ns) // of the exception in the calling interpreter. typedef struct _sharedexception { - char *name; - char *msg; + const char *name; + const char *msg; } _sharedexception; -static _sharedexception * -_sharedexception_new(void) -{ - _sharedexception *err = PyMem_NEW(_sharedexception, 1); - if (err == NULL) { - PyErr_NoMemory(); - return NULL; - } - err->name = NULL; - err->msg = NULL; - return err; -} +static const struct _sharedexception no_exception = { + .name = NULL, + .msg = NULL, +}; static void _sharedexception_clear(_sharedexception *exc) { if (exc->name != NULL) { - PyMem_Free(exc->name); + PyMem_RawFree((void *)exc->name); } if (exc->msg != NULL) { - PyMem_Free(exc->msg); + PyMem_RawFree((void *)exc->msg); } } -static void -_sharedexception_free(_sharedexception *exc) -{ - _sharedexception_clear(exc); - PyMem_Free(exc); -} - -static _sharedexception * -_sharedexception_bind(PyObject *exc) +static const char * +_sharedexception_bind(PyObject *exc, _sharedexception *sharedexc) { assert(exc != NULL); - char *failure = NULL; - - _sharedexception *err = _sharedexception_new(); - if (err == NULL) { - goto finally; - } + const char *failure = NULL; - PyObject *name = PyUnicode_FromFormat("%S", Py_TYPE(exc)); - if (name == NULL) { + PyObject *nameobj = PyUnicode_FromFormat("%S", Py_TYPE(exc)); + if (nameobj == NULL) { failure = "unable to format exception type name"; - goto finally; + goto error; } - err->name = _copy_raw_string(name); - Py_DECREF(name); - if (err->name == NULL) { + sharedexc->name = _copy_raw_string(nameobj); + Py_DECREF(nameobj); + if (sharedexc->name == NULL) { if (PyErr_ExceptionMatches(PyExc_MemoryError)) { failure = "out of memory copying exception type name"; } else { failure = "unable to encode and copy exception type name"; } - goto finally; + goto error; } if (exc != NULL) { - PyObject *msg = PyUnicode_FromFormat("%S", exc); - if (msg == NULL) { + PyObject *msgobj = PyUnicode_FromFormat("%S", exc); + if (msgobj == NULL) { failure = "unable to format exception message"; - goto finally; + goto error; } - err->msg = _copy_raw_string(msg); - Py_DECREF(msg); - if (err->msg == NULL) { + sharedexc->msg = _copy_raw_string(msgobj); + Py_DECREF(msgobj); + if (sharedexc->msg == NULL) { if (PyErr_ExceptionMatches(PyExc_MemoryError)) { failure = "out of memory copying exception message"; } else { failure = "unable to encode and copy exception message"; } - goto finally; + goto error; } } -finally: - if (failure != NULL) { - PyErr_Clear(); - if (err->name != NULL) { - PyMem_Free(err->name); - err->name = NULL; - } - err->msg = failure; - } - return err; + return NULL; + +error: + assert(failure != NULL); + PyErr_Clear(); + _sharedexception_clear(sharedexc); + *sharedexc = no_exception; + return failure; } static void @@ -430,7 +408,7 @@ _ensure_not_running(PyInterpreterState *interp) static int _run_script(PyInterpreterState *interp, const char *codestr, - _sharedns *shared, _sharedexception **exc) + _sharedns *shared, _sharedexception *sharedexc) { PyObject *excval = NULL; PyObject *main_mod = _PyInterpreterState_GetMainModule(interp); @@ -462,22 +440,20 @@ _run_script(PyInterpreterState *interp, const char *codestr, Py_DECREF(result); // We throw away the result. } - *exc = NULL; + *sharedexc = no_exception; return 0; error: excval = PyErr_GetRaisedException(); - _sharedexception *sharedexc = _sharedexception_bind(excval); - Py_XDECREF(excval); - if (sharedexc == NULL) { - fprintf(stderr, "RunFailedError: script raised an uncaught exception"); + const char *failure = _sharedexception_bind(excval, sharedexc); + if (failure != NULL) { + fprintf(stderr, + "RunFailedError: script raised an uncaught exception (%s)", + failure); PyErr_Clear(); - sharedexc = NULL; - } - else { - assert(!PyErr_Occurred()); } - *exc = sharedexc; + Py_XDECREF(excval); + assert(!PyErr_Occurred()); return -1; } @@ -505,7 +481,7 @@ _run_script_in_interpreter(PyObject *mod, PyInterpreterState *interp, } // Run the script. - _sharedexception *exc = NULL; + _sharedexception exc; int result = _run_script(interp, codestr, shared, &exc); // Switch back. @@ -514,10 +490,9 @@ _run_script_in_interpreter(PyObject *mod, PyInterpreterState *interp, } // Propagate any exception out to the caller. - if (exc != NULL) { + if (exc.name != NULL) { assert(state != NULL); - _sharedexception_apply(exc, state->RunFailedError); - _sharedexception_free(exc); + _sharedexception_apply(&exc, state->RunFailedError); } else if (result != 0) { // We were unable to allocate a shared exception. From 392f2ad3cbf2e1f24656fe0410a9b65882257582 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 13 Mar 2023 15:08:45 -0700 Subject: [PATCH 81/97] gh-98831: Use DECREF_INPUTS() more (#102409) --- Python/bytecodes.c | 80 +++++++++++++++----------------------- Python/generated_cases.c.h | 15 ++++--- 2 files changed, 38 insertions(+), 57 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a715bfd3289ec94..74582ecbbda103f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -384,8 +384,7 @@ dummy_func( if (!_PyErr_Occurred(tstate)) { _PyErr_SetKeyError(sub); } - Py_DECREF(dict); - Py_DECREF(sub); + DECREF_INPUTS(); ERROR_IF(true, error); } Py_INCREF(res); // Do this before DECREF'ing dict, sub @@ -420,7 +419,7 @@ dummy_func( inst(SET_ADD, (set, unused[oparg-1], v -- set, unused[oparg-1])) { int err = PySet_Add(set, v); - Py_DECREF(v); + DECREF_INPUTS(); ERROR_IF(err, error); PREDICT(JUMP_BACKWARD); } @@ -899,7 +898,7 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION */ PyObject **top = stack_pointer + oparg - 1; int res = unpack_iterable(tstate, seq, oparg, -1, top); - Py_DECREF(seq); + DECREF_INPUTS(); ERROR_IF(res == 0, error); } @@ -910,7 +909,7 @@ dummy_func( STAT_INC(UNPACK_SEQUENCE, hit); values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); - Py_DECREF(seq); + DECREF_INPUTS(); } inst(UNPACK_SEQUENCE_TUPLE, (unused/1, seq -- values[oparg])) { @@ -921,7 +920,7 @@ dummy_func( for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - Py_DECREF(seq); + DECREF_INPUTS(); } inst(UNPACK_SEQUENCE_LIST, (unused/1, seq -- values[oparg])) { @@ -932,14 +931,14 @@ dummy_func( for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - Py_DECREF(seq); + DECREF_INPUTS(); } inst(UNPACK_EX, (seq -- unused[oparg & 0xFF], unused, unused[oparg >> 8])) { int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - Py_DECREF(seq); + DECREF_INPUTS(); ERROR_IF(res == 0, error); } @@ -967,22 +966,21 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, v); - Py_DECREF(v); - Py_DECREF(owner); + DECREF_INPUTS(); ERROR_IF(err, error); } inst(DELETE_ATTR, (owner --)) { PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, (PyObject *)NULL); - Py_DECREF(owner); + DECREF_INPUTS(); ERROR_IF(err, error); } inst(STORE_GLOBAL, (v --)) { PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); - Py_DECREF(v); + DECREF_INPUTS(); ERROR_IF(err, error); } @@ -1249,9 +1247,7 @@ dummy_func( inst(BUILD_STRING, (pieces[oparg] -- str)) { str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); - for (int i = 0; i < oparg; i++) { - Py_DECREF(pieces[i]); - } + DECREF_INPUTS(); ERROR_IF(str == NULL, error); } @@ -1314,10 +1310,7 @@ dummy_func( if (map == NULL) goto error; - for (int i = 0; i < oparg; i++) { - Py_DECREF(values[i*2]); - Py_DECREF(values[i*2+1]); - } + DECREF_INPUTS(); ERROR_IF(map == NULL, error); } @@ -1373,10 +1366,7 @@ dummy_func( map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); - Py_DECREF(keys); - for (int i = 0; i < oparg; i++) { - Py_DECREF(values[i]); - } + DECREF_INPUTS(); ERROR_IF(map == NULL, error); } @@ -1464,7 +1454,7 @@ dummy_func( NULL | meth | arg1 | ... | argN */ - Py_DECREF(owner); + DECREF_INPUTS(); ERROR_IF(meth == NULL, error); res2 = NULL; res = meth; @@ -1473,7 +1463,7 @@ dummy_func( else { /* Classic, pushes one value. */ res = PyObject_GetAttr(owner, name); - Py_DECREF(owner); + DECREF_INPUTS(); ERROR_IF(res == NULL, error); } } @@ -1492,7 +1482,7 @@ dummy_func( STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - Py_DECREF(owner); + DECREF_INPUTS(); } inst(LOAD_ATTR_MODULE, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) { @@ -1509,7 +1499,7 @@ dummy_func( STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - Py_DECREF(owner); + DECREF_INPUTS(); } inst(LOAD_ATTR_WITH_HINT, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) { @@ -1540,7 +1530,7 @@ dummy_func( STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - Py_DECREF(owner); + DECREF_INPUTS(); } inst(LOAD_ATTR_SLOT, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) { @@ -1554,7 +1544,7 @@ dummy_func( STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - Py_DECREF(owner); + DECREF_INPUTS(); } inst(LOAD_ATTR_CLASS, (unused/1, type_version/2, unused/2, descr/4, cls -- res2 if (oparg & 1), res)) { @@ -1570,7 +1560,7 @@ dummy_func( res = descr; assert(res != NULL); Py_INCREF(res); - Py_DECREF(cls); + DECREF_INPUTS(); } inst(LOAD_ATTR_PROPERTY, (unused/1, type_version/2, func_version/2, fget/4, owner -- unused if (oparg & 1), unused)) { @@ -1707,8 +1697,7 @@ dummy_func( STAT_INC(COMPARE_OP, deferred); assert((oparg >> 4) <= Py_GE); res = PyObject_RichCompare(left, right, oparg>>4); - Py_DECREF(left); - Py_DECREF(right); + DECREF_INPUTS(); ERROR_IF(res == NULL, error); } @@ -1734,8 +1723,7 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 4) <= Py_GE); PyObject *cond = PyObject_RichCompare(left, right, oparg>>4); - Py_DECREF(left); - Py_DECREF(right); + DECREF_INPUTS(); ERROR_IF(cond == NULL, error); assert(next_instr[1].op.code == POP_JUMP_IF_FALSE || next_instr[1].op.code == POP_JUMP_IF_TRUE); @@ -1885,7 +1873,7 @@ dummy_func( } else { int err = PyObject_IsTrue(cond); - Py_DECREF(cond); + DECREF_INPUTS(); if (err == 0) { JUMPBY(oparg); } @@ -1905,7 +1893,7 @@ dummy_func( } else { int err = PyObject_IsTrue(cond); - Py_DECREF(cond); + DECREF_INPUTS(); if (err > 0) { JUMPBY(oparg); } @@ -1917,7 +1905,7 @@ dummy_func( inst(POP_JUMP_IF_NOT_NONE, (value -- )) { if (!Py_IsNone(value)) { - Py_DECREF(value); + DECREF_INPUTS(); JUMPBY(oparg); } else { @@ -1931,7 +1919,7 @@ dummy_func( JUMPBY(oparg); } else { - Py_DECREF(value); + DECREF_INPUTS(); } } @@ -2065,7 +2053,7 @@ dummy_func( if (iter == NULL) { goto error; } - Py_DECREF(iterable); + DECREF_INPUTS(); } PREDICT(LOAD_CONST); } @@ -2940,9 +2928,7 @@ dummy_func( assert(PyTuple_CheckExact(callargs)); result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing); - Py_DECREF(func); - Py_DECREF(callargs); - Py_XDECREF(kwargs); + DECREF_INPUTS(); assert(PEEK(3 + (oparg & 1)) == NULL); ERROR_IF(result == NULL, error); @@ -3009,9 +2995,7 @@ dummy_func( inst(BUILD_SLICE, (start, stop, step if (oparg == 3) -- slice)) { slice = PySlice_New(start, stop, step); - Py_DECREF(start); - Py_DECREF(stop); - Py_XDECREF(step); + DECREF_INPUTS(); ERROR_IF(slice == NULL, error); } @@ -3057,8 +3041,7 @@ dummy_func( } else { /* Actually call format(). */ result = PyObject_Format(value, fmt_spec); - Py_DECREF(value); - Py_XDECREF(fmt_spec); + DECREF_INPUTS(); ERROR_IF(result == NULL, error); } } @@ -3084,8 +3067,7 @@ dummy_func( assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - Py_DECREF(lhs); - Py_DECREF(rhs); + DECREF_INPUTS(); ERROR_IF(res == NULL, error); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 631b7844f9ed88c..b1dbb58d9563676 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1568,8 +1568,8 @@ PyObject **pieces = (stack_pointer - oparg); PyObject *str; str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); - for (int i = 0; i < oparg; i++) { - Py_DECREF(pieces[i]); + for (int _i = oparg; --_i >= 0;) { + Py_DECREF(pieces[_i]); } if (str == NULL) { STACK_SHRINK(oparg); goto error; } STACK_SHRINK(oparg); @@ -1665,9 +1665,8 @@ if (map == NULL) goto error; - for (int i = 0; i < oparg; i++) { - Py_DECREF(values[i*2]); - Py_DECREF(values[i*2+1]); + for (int _i = oparg*2; --_i >= 0;) { + Py_DECREF(values[_i]); } if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } STACK_SHRINK(oparg*2); @@ -1732,10 +1731,10 @@ map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); - Py_DECREF(keys); - for (int i = 0; i < oparg; i++) { - Py_DECREF(values[i]); + for (int _i = oparg; --_i >= 0;) { + Py_DECREF(values[_i]); } + Py_DECREF(keys); if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } STACK_SHRINK(oparg); stack_pointer[-1] = map; From 61479d46848bc7a7f9b571b0b09c4a4b4436d839 Mon Sep 17 00:00:00 2001 From: Blind4Basics <32236948+Blind4Basics@users.noreply.github.com> Date: Mon, 13 Mar 2023 23:35:37 +0100 Subject: [PATCH 82/97] gh-102627: Replace address pointing toward malicious web page (#102630) * Replace known bad address pointing toward a malicious web page. Co-authored-by: C.A.M. Gerlach Co-authored-by: Hugo van Kemenade --- Doc/library/concurrent.futures.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index c543c849585b7f0..09c9fc4e6e227a8 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -202,7 +202,7 @@ ThreadPoolExecutor Example 'http://www.cnn.com/', 'http://europe.wsj.com/', 'http://www.bbc.co.uk/', - 'http://some-made-up-domain.com/'] + 'http://nonexistant-subdomain.python.org/'] # Retrieve a single page and report the URL and contents def load_url(url, timeout): From 457e4d1a516c2b83edeff2f255f4cd6e7b114feb Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 13 Mar 2023 20:06:43 -0500 Subject: [PATCH 83/97] GH-102670: Use sumprod() to simplify, speed up, and improve accuracy of statistics functions (GH-102649) --- Lib/statistics.py | 26 ++++++++++--------- Lib/test/test_statistics.py | 12 ++++++++- ...-03-13-18-27-00.gh-issue-102670.GyoThv.rst | 2 ++ 3 files changed, 27 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-13-18-27-00.gh-issue-102670.GyoThv.rst diff --git a/Lib/statistics.py b/Lib/statistics.py index 7d5d750193a5abd..6bd214bbfe2ff59 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -1036,7 +1036,7 @@ def covariance(x, y, /): raise StatisticsError('covariance requires at least two data points') xbar = fsum(x) / n ybar = fsum(y) / n - sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y)) + sxy = sumprod((xi - xbar for xi in x), (yi - ybar for yi in y)) return sxy / (n - 1) @@ -1074,11 +1074,14 @@ def correlation(x, y, /, *, method='linear'): start = (n - 1) / -2 # Center rankings around zero x = _rank(x, start=start) y = _rank(y, start=start) - xbar = fsum(x) / n - ybar = fsum(y) / n - sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y)) - sxx = fsum((d := xi - xbar) * d for xi in x) - syy = fsum((d := yi - ybar) * d for yi in y) + else: + xbar = fsum(x) / n + ybar = fsum(y) / n + x = [xi - xbar for xi in x] + y = [yi - ybar for yi in y] + sxy = sumprod(x, y) + sxx = sumprod(x, x) + syy = sumprod(y, y) try: return sxy / sqrt(sxx * syy) except ZeroDivisionError: @@ -1131,14 +1134,13 @@ def linear_regression(x, y, /, *, proportional=False): raise StatisticsError('linear regression requires that both inputs have same number of data points') if n < 2: raise StatisticsError('linear regression requires at least two data points') - if proportional: - sxy = fsum(xi * yi for xi, yi in zip(x, y)) - sxx = fsum(xi * xi for xi in x) - else: + if not proportional: xbar = fsum(x) / n ybar = fsum(y) / n - sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y)) - sxx = fsum((d := xi - xbar) * d for xi in x) + x = [xi - xbar for xi in x] # List because used three times below + y = (yi - ybar for yi in y) # Generator because only used once below + sxy = sumprod(x, y) + 0.0 # Add zero to coerce result to a float + sxx = sumprod(x, x) try: slope = sxy / sxx # equivalent to: covariance(x, y) / variance(x) except ZeroDivisionError: diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 31a3cb6b53a6f2a..f0fa6454b1f91a7 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -1,4 +1,4 @@ -"""Test suite for statistics module, including helper NumericTestCase and +x = """Test suite for statistics module, including helper NumericTestCase and approx_equal function. """ @@ -2610,6 +2610,16 @@ def test_proportional(self): self.assertAlmostEqual(slope, 20 + 1/150) self.assertEqual(intercept, 0.0) + def test_float_output(self): + x = [Fraction(2, 3), Fraction(3, 4)] + y = [Fraction(4, 5), Fraction(5, 6)] + slope, intercept = statistics.linear_regression(x, y) + self.assertTrue(isinstance(slope, float)) + self.assertTrue(isinstance(intercept, float)) + slope, intercept = statistics.linear_regression(x, y, proportional=True) + self.assertTrue(isinstance(slope, float)) + self.assertTrue(isinstance(intercept, float)) + class TestNormalDist: # General note on precision: The pdf(), cdf(), and overlap() methods diff --git a/Misc/NEWS.d/next/Library/2023-03-13-18-27-00.gh-issue-102670.GyoThv.rst b/Misc/NEWS.d/next/Library/2023-03-13-18-27-00.gh-issue-102670.GyoThv.rst new file mode 100644 index 000000000000000..3de09f86754f3ec --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-13-18-27-00.gh-issue-102670.GyoThv.rst @@ -0,0 +1,2 @@ +Optimized fmean(), correlation(), covariance(), and linear_regression() +using the new math.sumprod() function. From cbd3fbfb6e5c1cc96bbeb99483a580f165b01671 Mon Sep 17 00:00:00 2001 From: Jacob Bower <1978924+jbower-fb@users.noreply.github.com> Date: Mon, 13 Mar 2023 20:35:54 -0500 Subject: [PATCH 84/97] gh-102013: Add PyUnstable_GC_VisitObjects (#102014) --- Doc/c-api/gcsupport.rst | 33 +++++++++ Include/objimpl.h | 19 +++++ ...-02-18-00-55-14.gh-issue-102013.83mrtI.rst | 1 + Modules/_testcapimodule.c | 69 +++++++++++++++++++ Modules/gcmodule.c | 24 +++++++ 5 files changed, 146 insertions(+) create mode 100644 Misc/NEWS.d/next/C API/2023-02-18-00-55-14.gh-issue-102013.83mrtI.rst diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index 8c90d1e8991c100..cb5d64a50487fe6 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -228,3 +228,36 @@ garbage collection runs. Returns the current state, 0 for disabled and 1 for enabled. .. versionadded:: 3.10 + + +Querying Garbage Collector State +-------------------------------- + +The C-API provides the following interface for querying information about +the garbage collector. + +.. c:function:: void PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void *arg) + + Run supplied *callback* on all live GC-capable objects. *arg* is passed through to + all invocations of *callback*. + + .. warning:: + If new objects are (de)allocated by the callback it is undefined if they + will be visited. + + Garbage collection is disabled during operation. Explicitly running a collection + in the callback may lead to undefined behaviour e.g. visiting the same objects + multiple times or not at all. + + .. versionadded:: 3.12 + +.. c:type:: int (*gcvisitobjects_t)(PyObject *object, void *arg) + + Type of the visitor function to be passed to :c:func:`PyUnstable_GC_VisitObjects`. + *arg* is the same as the *arg* passed to ``PyUnstable_GC_VisitObjects``. + Return ``0`` to continue iteration, return ``1`` to stop iteration. Other return + values are reserved for now so behavior on returning anything else is undefined. + + .. versionadded:: 3.12 + + diff --git a/Include/objimpl.h b/Include/objimpl.h index dde8df34835328e..ef871c5ea93ebee 100644 --- a/Include/objimpl.h +++ b/Include/objimpl.h @@ -157,6 +157,25 @@ PyAPI_FUNC(int) PyGC_Enable(void); PyAPI_FUNC(int) PyGC_Disable(void); PyAPI_FUNC(int) PyGC_IsEnabled(void); + +#if !defined(Py_LIMITED_API) +/* Visit all live GC-capable objects, similar to gc.get_objects(None). The + * supplied callback is called on every such object with the void* arg set + * to the supplied arg. Returning 0 from the callback ends iteration, returning + * 1 allows iteration to continue. Returning any other value may result in + * undefined behaviour. + * + * If new objects are (de)allocated by the callback it is undefined if they + * will be visited. + + * Garbage collection is disabled during operation. Explicitly running a + * collection in the callback may lead to undefined behaviour e.g. visiting the + * same objects multiple times or not at all. + */ +typedef int (*gcvisitobjects_t)(PyObject*, void*); +PyAPI_FUNC(void) PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void* arg); +#endif + /* Test if a type has a GC head */ #define PyType_IS_GC(t) PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) diff --git a/Misc/NEWS.d/next/C API/2023-02-18-00-55-14.gh-issue-102013.83mrtI.rst b/Misc/NEWS.d/next/C API/2023-02-18-00-55-14.gh-issue-102013.83mrtI.rst new file mode 100644 index 000000000000000..0350237ebc73907 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-02-18-00-55-14.gh-issue-102013.83mrtI.rst @@ -0,0 +1 @@ +Add a new (unstable) C-API function for iterating over GC'able objects using a callback: ``PyUnstable_VisitObjects``. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index ea67017a1ba3b16..f45d0312e944119 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3310,6 +3310,73 @@ function_set_kw_defaults(PyObject *self, PyObject *args) Py_RETURN_NONE; } +struct gc_visit_state_basic { + PyObject *target; + int found; +}; + +static int +gc_visit_callback_basic(PyObject *obj, void *arg) +{ + struct gc_visit_state_basic *state = (struct gc_visit_state_basic *)arg; + if (obj == state->target) { + state->found = 1; + return 0; + } + return 1; +} + +static PyObject * +test_gc_visit_objects_basic(PyObject *Py_UNUSED(self), + PyObject *Py_UNUSED(ignored)) +{ + PyObject *obj; + struct gc_visit_state_basic state; + + obj = PyList_New(0); + if (obj == NULL) { + return NULL; + } + state.target = obj; + state.found = 0; + + PyUnstable_GC_VisitObjects(gc_visit_callback_basic, &state); + Py_DECREF(obj); + if (!state.found) { + PyErr_SetString( + PyExc_AssertionError, + "test_gc_visit_objects_basic: Didn't find live list"); + return NULL; + } + Py_RETURN_NONE; +} + +static int +gc_visit_callback_exit_early(PyObject *obj, void *arg) + { + int *visited_i = (int *)arg; + (*visited_i)++; + if (*visited_i == 2) { + return 0; + } + return 1; +} + +static PyObject * +test_gc_visit_objects_exit_early(PyObject *Py_UNUSED(self), + PyObject *Py_UNUSED(ignored)) +{ + int visited_i = 0; + PyUnstable_GC_VisitObjects(gc_visit_callback_exit_early, &visited_i); + if (visited_i != 2) { + PyErr_SetString( + PyExc_AssertionError, + "test_gc_visit_objects_exit_early: did not exit when expected"); + } + Py_RETURN_NONE; +} + + static PyObject *test_buildvalue_issue38913(PyObject *, PyObject *); static PyMethodDef TestMethods[] = { @@ -3452,6 +3519,8 @@ static PyMethodDef TestMethods[] = { {"function_set_defaults", function_set_defaults, METH_VARARGS, NULL}, {"function_get_kw_defaults", function_get_kw_defaults, METH_O, NULL}, {"function_set_kw_defaults", function_set_kw_defaults, METH_VARARGS, NULL}, + {"test_gc_visit_objects_basic", test_gc_visit_objects_basic, METH_NOARGS, NULL}, + {"test_gc_visit_objects_exit_early", test_gc_visit_objects_exit_early, METH_NOARGS, NULL}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 5879c5e11fe14a0..4eaa5490b6134c7 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -2401,3 +2401,27 @@ PyObject_GC_IsFinalized(PyObject *obj) } return 0; } + +void +PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void *arg) +{ + size_t i; + GCState *gcstate = get_gc_state(); + int origenstate = gcstate->enabled; + gcstate->enabled = 0; + for (i = 0; i < NUM_GENERATIONS; i++) { + PyGC_Head *gc_list, *gc; + gc_list = GEN_HEAD(gcstate, i); + for (gc = GC_NEXT(gc_list); gc != gc_list; gc = GC_NEXT(gc)) { + PyObject *op = FROM_GC(gc); + Py_INCREF(op); + int res = callback(op, arg); + Py_DECREF(op); + if (!res) { + goto done; + } + } + } +done: + gcstate->enabled = origenstate; +} From 7bdb331b67c4aee513e01794ba7dd2c3be43d3ca Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Tue, 14 Mar 2023 11:07:59 +0900 Subject: [PATCH 85/97] doc: Remove a duplicate 'versionchanged' in library/asyncio-task (gh-102677) --- Doc/library/asyncio-task.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 9b9842432822683..5f1449e1b599efb 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -814,9 +814,6 @@ Waiting Primitives Raises :exc:`TimeoutError` if the timeout occurs before all Futures are done. - .. versionchanged:: 3.10 - Removed the *loop* parameter. - Example:: for coro in as_completed(aws): From 3d872a74c8c16d4a077c2223f678b1f8f7e0e988 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Tue, 14 Mar 2023 14:22:21 +0530 Subject: [PATCH 86/97] GH-100227: cleanup initialization of global interned dict (#102682) --- Objects/unicodeobject.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 2d50f9c340f2f36..b9fb53147b9b511 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -14533,6 +14533,15 @@ _PyUnicode_InitGlobalObjects(PyInterpreterState *interp) return _PyStatus_OK(); } + // Initialize the global interned dict + PyObject *interned = PyDict_New(); + if (interned == NULL) { + PyErr_Clear(); + return _PyStatus_ERR("failed to create interned dict"); + } + + set_interned_dict(interned); + /* Intern statically allocated string identifiers and deepfreeze strings. * This must be done before any module initialization so that statically * allocated string identifiers are used instead of heap allocated strings. @@ -14600,14 +14609,7 @@ PyUnicode_InternInPlace(PyObject **p) } PyObject *interned = get_interned_dict(); - if (interned == NULL) { - interned = PyDict_New(); - if (interned == NULL) { - PyErr_Clear(); /* Don't leave an exception */ - return; - } - set_interned_dict(interned); - } + assert(interned != NULL); PyObject *t = PyDict_SetDefault(interned, s, s); if (t == NULL) { From 88c262c086077377b40dfae5e46f597e28ffe3c9 Mon Sep 17 00:00:00 2001 From: T Date: Tue, 14 Mar 2023 17:23:52 +0800 Subject: [PATCH 87/97] gh-100315: clarification to `__slots__` docs. (#102621) refer to tp_itemsize in discussion on "variable-length" built-in types --- Doc/reference/datamodel.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index f447bbb1216d52d..1865d09fcaa1279 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1944,8 +1944,10 @@ Notes on using *__slots__* descriptor directly from the base class). This renders the meaning of the program undefined. In the future, a check may be added to prevent this. -* Nonempty *__slots__* does not work for classes derived from "variable-length" - built-in types such as :class:`int`, :class:`bytes` and :class:`tuple`. +* :exc:`TypeError` will be raised if nonempty *__slots__* are defined for a + class derived from a + :c:member:`"variable-length" built-in type ` such as + :class:`int`, :class:`bytes`, and :class:`tuple`. * Any non-string :term:`iterable` may be assigned to *__slots__*. From a703f743dbf2675948e59c44fa9d7112f7825100 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Tue, 14 Mar 2023 15:21:38 +0530 Subject: [PATCH 88/97] GH-94851: check unicode consistency of static strings in debug mode (#102684) --- .../internal/pycore_unicodeobject_generated.h | 664 ++++++++++++++++++ Tools/build/generate_global_objects.py | 1 + 2 files changed, 665 insertions(+) diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 52af37a8e60aa8e..fea9b6dbb1a75f0 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -13,1332 +13,1996 @@ static inline void _PyUnicode_InitStaticStrings(void) { PyObject *string; string = &_Py_ID(CANCELLED); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(FINISHED); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(False); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(JSONDecodeError); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(PENDING); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(Py_Repr); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(TextIOWrapper); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(True); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(WarningMessage); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_WindowsConsoleIO); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__IOBase_closed); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__abc_tpflags__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__abs__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__abstractmethods__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__add__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__aenter__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__aexit__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__aiter__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__all__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__and__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__anext__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__annotations__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__args__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__asyncio_running_event_loop__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__await__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__bases__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__bool__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__build_class__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__builtins__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__bytes__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__call__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__cantrace__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__class__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__class_getitem__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__classcell__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__complex__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__contains__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__copy__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__ctypes_from_outparam__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__del__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__delattr__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__delete__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__delitem__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__dict__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__dictoffset__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__dir__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__divmod__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__doc__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__enter__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__eq__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__exit__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__file__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__float__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__floordiv__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__format__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__fspath__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__ge__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__get__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__getattr__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__getattribute__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__getinitargs__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__getitem__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__getnewargs__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__getnewargs_ex__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__getstate__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__gt__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__hash__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__iadd__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__iand__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__ifloordiv__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__ilshift__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__imatmul__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__imod__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__import__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__imul__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__index__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__init__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__init_subclass__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__instancecheck__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__int__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__invert__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__ior__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__ipow__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__irshift__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__isabstractmethod__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__isub__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__iter__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__itruediv__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__ixor__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__le__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__len__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__length_hint__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__lltrace__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__loader__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__lshift__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__lt__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__main__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__matmul__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__missing__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__mod__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__module__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__mro_entries__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__mul__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__name__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__ne__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__neg__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__new__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__newobj__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__newobj_ex__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__next__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__notes__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__or__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__orig_class__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__origin__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__package__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__parameters__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__path__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__pos__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__pow__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__prepare__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__qualname__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__radd__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__rand__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__rdivmod__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__reduce__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__reduce_ex__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__repr__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__reversed__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__rfloordiv__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__rlshift__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__rmatmul__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__rmod__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__rmul__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__ror__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__round__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__rpow__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__rrshift__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__rshift__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__rsub__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__rtruediv__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__rxor__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__set__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__set_name__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__setattr__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__setitem__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__setstate__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__sizeof__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__slotnames__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__slots__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__spec__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__str__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__sub__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__subclasscheck__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__subclasshook__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__truediv__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__trunc__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__typing_is_unpacked_typevartuple__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__typing_prepare_subst__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__typing_subst__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__typing_unpacked_tuple_args__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__warningregistry__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__weaklistoffset__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__weakref__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(__xor__); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_abc_impl); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_abstract_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_active); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_annotation); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_anonymous_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_argtypes_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_as_parameter_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_asyncio_future_blocking); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_blksize); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_bootstrap); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_check_retval_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_dealloc_warn); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_feature_version); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_fields_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_finalizing); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_find_and_load); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_fix_up_module); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_flags_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_get_sourcefile); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_handle_fromlist); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_initializing); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_io); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_is_text_encoding); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_length_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_limbo); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_lock_unlock_module); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_loop); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_needs_com_addref_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_pack_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_restype_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_showwarnmsg); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_shutdown); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_slotnames); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_strptime_datetime); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_swappedbytes_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_type_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_uninitialized_submodules); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_warn_unawaited_coroutine); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_xoptions); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(a); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(abs_tol); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(access); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(add); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(add_done_callback); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(after_in_child); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(after_in_parent); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(aggregate_class); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(append); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(argdefs); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(arguments); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(argv); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(as_integer_ratio); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(ast); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(attribute); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(authorizer_callback); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(autocommit); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(b); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(backtick); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(base); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(before); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(big); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(binary_form); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(block); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(buffer); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(buffer_callback); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(buffer_size); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(buffering); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(buffers); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(bufsize); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(builtins); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(byteorder); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(bytes); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(bytes_per_sep); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(c); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(c_call); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(c_exception); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(c_return); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(cached_statements); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(cadata); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(cafile); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(call); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(call_exception_handler); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(call_soon); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(cancel); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(capath); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(category); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(cb_type); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(certfile); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(check_same_thread); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(clear); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(close); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(closed); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(closefd); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(closure); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_argcount); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_cellvars); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_code); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_consts); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_exceptiontable); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_filename); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_firstlineno); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_flags); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_freevars); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_kwonlyargcount); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_linetable); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_name); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_names); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_nlocals); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_posonlyargcount); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_qualname); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_stacksize); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(co_varnames); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(code); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(command); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(comment_factory); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(consts); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(context); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(cookie); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(copy); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(copyreg); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(coro); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(count); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(cwd); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(d); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(data); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(database); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(decode); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(decoder); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(default); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(defaultaction); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(delete); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(depth); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(detect_types); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(deterministic); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(device); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(dict); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(dictcomp); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(difference_update); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(digest); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(digest_size); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(digestmod); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(dir_fd); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(discard); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(dispatch_table); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(displayhook); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(dklen); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(doc); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(dont_inherit); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(dst); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(dst_dir_fd); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(duration); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(e); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(effective_ids); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(element_factory); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(encode); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(encoding); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(end); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(end_lineno); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(end_offset); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(endpos); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(env); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(errors); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(event); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(eventmask); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(exc_type); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(exc_value); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(excepthook); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(exception); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(exp); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(extend); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(facility); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(factory); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(false); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(family); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(fanout); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(fd); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(fd2); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(fdel); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(fget); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(file); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(file_actions); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(filename); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(fileno); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(filepath); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(fillvalue); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(filters); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(final); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(find_class); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(fix_imports); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(flags); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(flush); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(follow_symlinks); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(format); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(frequency); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(from_param); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(fromlist); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(fromtimestamp); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(fromutc); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(fset); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(func); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(future); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(generation); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(genexpr); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(get); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(get_debug); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(get_event_loop); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(get_loop); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(get_source); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(getattr); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(getstate); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(gid); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(globals); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(groupindex); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(groups); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(handle); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(hash_name); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(header); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(headers); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(hi); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(hook); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(id); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(ident); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(ignore); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(imag); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(importlib); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(in_fd); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(incoming); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(indexgroup); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(inf); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(inheritable); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(initial); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(initial_bytes); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(initial_value); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(initval); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(inner_size); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(input); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(insert_comments); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(insert_pis); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(instructions); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(intern); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(intersection); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(isatty); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(isinstance); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(isoformat); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(isolation_level); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(istext); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(item); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(items); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(iter); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(iterable); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(iterations); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(join); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(jump); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(keepends); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(key); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(keyfile); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(keys); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(kind); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(kw); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(kw1); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(kw2); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(lambda); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(last); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(last_node); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(last_traceback); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(last_type); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(last_value); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(latin1); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(leaf_size); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(len); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(length); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(level); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(limit); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(line); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(line_buffering); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(lineno); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(listcomp); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(little); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(lo); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(locale); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(locals); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(logoption); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(loop); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(mapping); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(match); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(max_length); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(maxdigits); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(maxevents); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(maxmem); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(maxsplit); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(maxvalue); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(memLevel); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(memlimit); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(message); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(metaclass); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(method); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(mod); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(mode); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(module); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(module_globals); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(modules); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(mro); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(msg); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(mycmp); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(n); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(n_arg); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(n_fields); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(n_sequence_fields); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(n_unnamed_fields); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(name); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(name_from); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(namespace_separator); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(namespaces); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(narg); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(ndigits); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(new_limit); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(newline); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(newlines); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(next); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(node_depth); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(node_offset); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(ns); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(nstype); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(nt); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(null); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(number); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(obj); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(object); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(offset); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(offset_dst); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(offset_src); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(on_type_read); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(onceregistry); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(only_keys); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(oparg); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(opcode); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(open); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(opener); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(operation); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(optimize); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(options); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(order); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(out_fd); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(outgoing); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(overlapped); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(owner); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(p); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(pages); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(parent); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(password); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(path); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(pattern); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(peek); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(persistent_id); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(persistent_load); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(person); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(pi_factory); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(pid); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(policy); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(pos); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(pos1); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(pos2); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(posix); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(print_file_and_line); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(priority); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(progress); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(progress_handler); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(proto); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(protocol); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(ps1); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(ps2); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(query); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(quotetabs); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(r); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(raw); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(read); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(read1); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(readable); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(readall); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(readinto); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(readinto1); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(readline); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(readonly); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(real); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(reducer_override); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(registry); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(rel_tol); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(reload); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(repl); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(replace); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(reserved); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(reset); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(resetids); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(return); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(reverse); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(reversed); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(s); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(salt); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(sched_priority); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(scheduler); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(seek); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(seekable); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(selectors); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(self); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(send); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(sep); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(sequence); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(server_hostname); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(server_side); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(session); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(setcomp); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(setpgroup); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(setsid); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(setsigdef); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(setsigmask); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(setstate); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(shape); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(show_cmd); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(signed); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(size); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(sizehint); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(skip_file_prefixes); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(sleep); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(sock); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(sort); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(sound); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(source); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(source_traceback); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(src); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(src_dir_fd); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(stacklevel); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(start); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(statement); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(status); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(stderr); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(stdin); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(stdout); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(step); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(store_name); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(strategy); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(strftime); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(strict); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(strict_mode); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(string); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(sub_key); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(symmetric_difference_update); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(tabsize); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(tag); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(target); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(target_is_directory); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(task); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(tb_frame); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(tb_lasti); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(tb_lineno); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(tb_next); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(tell); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(template); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(term); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(text); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(threading); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(throw); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(timeout); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(times); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(timetuple); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(top); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(trace_callback); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(traceback); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(trailers); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(translate); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(true); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(truncate); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(twice); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(txt); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(type); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(tz); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(tzname); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(uid); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(unlink); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(unraisablehook); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(uri); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(usedforsecurity); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(value); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(values); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(version); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(volume); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(warnings); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(warnoptions); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(wbits); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(week); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(weekday); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(which); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(who); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(withdata); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(writable); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(write); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(write_through); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(x); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(year); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(zdict); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); } /* End auto-generated code */ diff --git a/Tools/build/generate_global_objects.py b/Tools/build/generate_global_objects.py index 9ceae89878cda82..1f53f02d41ef399 100644 --- a/Tools/build/generate_global_objects.py +++ b/Tools/build/generate_global_objects.py @@ -360,6 +360,7 @@ def generate_static_strings_initializer(identifiers, strings): # This use of _Py_ID() is ignored by iter_global_strings() # since iter_files() ignores .h files. printer.write(f'string = &_Py_ID({i});') + printer.write(f'assert(_PyUnicode_CheckConsistency(string, 1));') printer.write(f'PyUnicode_InternInPlace(&string);') # XXX What about "strings"? printer.write(END) From 1ff81c0cb67215694f084e51c4d35ae53b9f5cf9 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 14 Mar 2023 10:05:54 -0600 Subject: [PATCH 89/97] gh-81057: Add a CI Check for New Unsupported C Global Variables (gh-102506) This will keep us from adding new unsupported (i.e. non-const) C global variables, which would break interpreter isolation. FYI, historically it is very uncommon for new global variables to get added. Furthermore, it is rare for new code to break the c-analyzer. So the check should almost always pass unnoticed. Note that I've removed test_check_c_globals. A test wasn't a great fit conceptually and was super slow on debug builds. A CI check is a better fit. This also resolves gh-100237. https://github.com/python/cpython/issues/81057 --- .github/workflows/build.yml | 3 + Lib/test/test_check_c_globals.py | 34 ---------- Makefile.pre.in | 6 ++ .../c_parser/preprocessor/common.py | 6 +- Tools/c-analyzer/c_parser/preprocessor/gcc.py | 23 ++++--- Tools/c-analyzer/cpython/__main__.py | 62 ++++++++++++++++--- Tools/c-analyzer/cpython/_parser.py | 9 ++- Tools/c-analyzer/cpython/ignored.tsv | 1 + 8 files changed, 90 insertions(+), 54 deletions(-) delete mode 100644 Lib/test/test_check_c_globals.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2241b0b8aa409e0..4e5328282f12242 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -111,6 +111,9 @@ jobs: run: make smelly - name: Check limited ABI symbols run: make check-limited-abi + - name: Check for unsupported C global variables + if: github.event_name == 'pull_request' # $GITHUB_EVENT_NAME + run: make check-c-globals build_win32: name: 'Windows (x86)' diff --git a/Lib/test/test_check_c_globals.py b/Lib/test/test_check_c_globals.py deleted file mode 100644 index 670be52422f799d..000000000000000 --- a/Lib/test/test_check_c_globals.py +++ /dev/null @@ -1,34 +0,0 @@ -import unittest -import test.test_tools -from test.support.warnings_helper import save_restore_warnings_filters - - -# TODO: gh-92584: c-analyzer uses distutils which was removed in Python 3.12 -raise unittest.SkipTest("distutils has been removed in Python 3.12") - - -test.test_tools.skip_if_missing('c-analyzer') -with test.test_tools.imports_under_tool('c-analyzer'): - # gh-95349: Save/restore warnings filters to leave them unchanged. - # Importing the c-analyzer imports docutils which imports pkg_resources - # which adds a warnings filter. - with save_restore_warnings_filters(): - from cpython.__main__ import main - - -class ActualChecks(unittest.TestCase): - - # XXX Also run the check in "make check". - #@unittest.expectedFailure - # Failing on one of the buildbots (see https://bugs.python.org/issue36876). - @unittest.skip('activate this once all the globals have been resolved') - def test_check_c_globals(self): - try: - main('check', {}) - except NotImplementedError: - raise unittest.SkipTest('not supported on this host') - - -if __name__ == '__main__': - # Test needs to be a package, so we can do relative imports. - unittest.main() diff --git a/Makefile.pre.in b/Makefile.pre.in index 1a1853bf3d78712..59762165c100367 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2560,6 +2560,12 @@ distclean: clobber docclean smelly: all $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/build/smelly.py +# Check if any unsupported C global variables have been added. +check-c-globals: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/c-analyzer/check-c-globals.py \ + --format summary \ + --traceback + # Find files with funny names funny: find $(SUBDIRS) $(SUBDIRSTOO) \ diff --git a/Tools/c-analyzer/c_parser/preprocessor/common.py b/Tools/c-analyzer/c_parser/preprocessor/common.py index 4291a066337545a..dbe1edeef385276 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/common.py +++ b/Tools/c-analyzer/c_parser/preprocessor/common.py @@ -115,15 +115,15 @@ def converted_error(tool, argv, filename): def convert_error(tool, argv, filename, stderr, rc): error = (stderr.splitlines()[0], rc) if (_expected := is_os_mismatch(filename, stderr)): - logger.debug(stderr.strip()) + logger.info(stderr.strip()) raise OSMismatchError(filename, _expected, argv, error, tool) elif (_missing := is_missing_dep(stderr)): - logger.debug(stderr.strip()) + logger.info(stderr.strip()) raise MissingDependenciesError(filename, (_missing,), argv, error, tool) elif '#error' in stderr: # XXX Ignore incompatible files. error = (stderr.splitlines()[1], rc) - logger.debug(stderr.strip()) + logger.info(stderr.strip()) raise ErrorDirectiveError(filename, argv, error, tool) else: # Try one more time, with stderr written to the terminal. diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index 7ef1a8afc3b135e..24c1b0e9b9d48c7 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -6,6 +6,11 @@ TOOL = 'gcc' +META_FILES = { + '', + '', +} + # https://gcc.gnu.org/onlinedocs/cpp/Preprocessor-Output.html # flags: # 1 start of a new file @@ -75,11 +80,15 @@ def _iter_lines(text, reqfile, samefiles, cwd, raw=False): # The first line is special. # The next two lines are consistent. - for expected in [ - f'# 1 "{reqfile}"', - '# 1 ""', - '# 1 ""', - ]: + firstlines = [ + f'# 0 "{reqfile}"', + '# 0 ""', + '# 0 ""', + ] + if text.startswith('# 1 '): + # Some preprocessors emit a lineno of 1 for line-less entries. + firstlines = [l.replace('# 0 ', '# 1 ') for l in firstlines] + for expected in firstlines: line = next(lines) if line != expected: raise NotImplementedError((line, expected)) @@ -121,7 +130,7 @@ def _iter_top_include_lines(lines, topfile, cwd, # _parse_marker_line() that the preprocessor reported lno as 1. lno = 1 for line in lines: - if line == '# 1 "" 2': + if line == '# 0 "" 2' or line == '# 1 "" 2': # We're done with this top-level include. return @@ -174,8 +183,8 @@ def _parse_marker_line(line, reqfile=None): return None, None, None lno, origfile, flags = m.groups() lno = int(lno) + assert origfile not in META_FILES, (line,) assert lno > 0, (line, lno) - assert origfile not in ('', ''), (line,) flags = set(int(f) for f in flags.split()) if flags else () if 1 in flags: diff --git a/Tools/c-analyzer/cpython/__main__.py b/Tools/c-analyzer/cpython/__main__.py index 2b9e4233b95ac46..fe7a16726f45a9d 100644 --- a/Tools/c-analyzer/cpython/__main__.py +++ b/Tools/c-analyzer/cpython/__main__.py @@ -1,5 +1,6 @@ import logging import sys +import textwrap from c_common.fsutil import expand_filenames, iter_files_by_suffix from c_common.scriptutil import ( @@ -26,6 +27,39 @@ logger = logging.getLogger(__name__) +CHECK_EXPLANATION = textwrap.dedent(''' + ------------------------- + + Non-constant global variables are generally not supported + in the CPython repo. We use a tool to analyze the C code + and report if any unsupported globals are found. The tool + may be run manually with: + + ./python Tools/c-analyzer/check-c-globals.py --format summary [FILE] + + Occasionally the tool is unable to parse updated code. + If this happens then add the file to the "EXCLUDED" list + in Tools/c-analyzer/cpython/_parser.py and create a new + issue for fixing the tool (and CC ericsnowcurrently + on the issue). + + If the tool reports an unsupported global variable and + it is actually const (and thus supported) then first try + fixing the declaration appropriately in the code. If that + doesn't work then add the variable to the "should be const" + section of Tools/c-analyzer/cpython/ignored.tsv. + + If the tool otherwise reports an unsupported global variable + then first try to make it non-global, possibly adding to + PyInterpreterState (for core code) or module state (for + extension modules). In an emergency, you can add the + variable to Tools/c-analyzer/cpython/globals-to-fix.tsv + to get CI passing, but doing so should be avoided. If + this course it taken, be sure to create an issue for + eliminating the global (and CC ericsnowcurrently). +''') + + def _resolve_filenames(filenames): if filenames: resolved = (_files.resolve_filename(f) for f in filenames) @@ -123,14 +157,26 @@ def _cli_check(parser, **kwargs): def cmd_check(filenames=None, **kwargs): filenames = _resolve_filenames(filenames) kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print) - c_analyzer.cmd_check( - filenames, - relroot=REPO_ROOT, - _analyze=_analyzer.analyze, - _CHECKS=CHECKS, - file_maxsizes=_parser.MAX_SIZES, - **kwargs - ) + try: + c_analyzer.cmd_check( + filenames, + relroot=REPO_ROOT, + _analyze=_analyzer.analyze, + _CHECKS=CHECKS, + file_maxsizes=_parser.MAX_SIZES, + **kwargs + ) + except SystemExit as exc: + num_failed = exc.args[0] if getattr(exc, 'args', None) else None + if isinstance(num_failed, int): + if num_failed > 0: + sys.stderr.flush() + print(CHECK_EXPLANATION, flush=True) + raise # re-raise + except Exception: + sys.stderr.flush() + print(CHECK_EXPLANATION, flush=True) + raise # re-raise def cmd_analyze(filenames=None, **kwargs): diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index e7764165d36c4c0..6da4fbbf4224f17 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -106,15 +106,20 @@ def clean_lines(text): * ./Include/internal Modules/_decimal/**/*.c Modules/_decimal/libmpdec +Modules/_elementtree.c Modules/expat Modules/_hacl/*.c Modules/_hacl/include Modules/_hacl/*.h Modules/_hacl/include -Modules/_tkinter.c /usr/include/tcl8.6 Modules/md5module.c Modules/_hacl/include Modules/sha1module.c Modules/_hacl/include Modules/sha2module.c Modules/_hacl/include -Modules/tkappinit.c /usr/include/tcl Objects/stringlib/*.h Objects +# possible system-installed headers, just in case +Modules/_tkinter.c /usr/include/tcl8.6 +Modules/_uuidmodule.c /usr/include/uuid +Modules/nismodule.c /usr/include/tirpc +Modules/tkappinit.c /usr/include/tcl + # @end=tsv@ ''')[1:] diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 700ddf2851839e1..048112dd9925557 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -228,6 +228,7 @@ Modules/_xxinterpchannelsmodule.c - _channelid_end_recv - Modules/_xxinterpchannelsmodule.c - _channelid_end_send - Modules/_zoneinfo.c - DAYS_BEFORE_MONTH - Modules/_zoneinfo.c - DAYS_IN_MONTH - +Modules/_xxsubinterpretersmodule.c - no_exception - Modules/arraymodule.c - descriptors - Modules/arraymodule.c - emptybuf - Modules/cjkcodecs/_codecs_cn.c - _mapping_list - From 80abd62647b2a36947a11a6a8e395061be6f0c61 Mon Sep 17 00:00:00 2001 From: Paul Watson Date: Tue, 14 Mar 2023 13:40:12 -0500 Subject: [PATCH 90/97] gh-102354: change python3 to python in docs examples (#102696) --- Doc/library/__main__.rst | 2 +- Doc/library/importlib.metadata.rst | 2 +- Doc/library/pdb.rst | 4 ++-- Doc/tutorial/venv.rst | 2 +- Doc/using/venv-create.inc | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/library/__main__.rst b/Doc/library/__main__.rst index 6a2a7a7317f7114..761c88710f98910 100644 --- a/Doc/library/__main__.rst +++ b/Doc/library/__main__.rst @@ -259,7 +259,7 @@ one mentioned below are preferred. See :mod:`venv` for an example of a package with a minimal ``__main__.py`` in the standard library. It doesn't contain a ``if __name__ == '__main__'`` - block. You can invoke it with ``python3 -m venv [directory]``. + block. You can invoke it with ``python -m venv [directory]``. See :mod:`runpy` for more details on the :option:`-m` flag to the interpreter executable. diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index 988d1a317f59602..6e084101995e259 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -73,7 +73,7 @@ something into it: .. code-block:: shell-session - $ python3 -m venv example + $ python -m venv example $ source example/bin/activate (example) $ python -m pip install wheel diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 4ae12a5d03a78db..21c6ca8622dceb8 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -58,7 +58,7 @@ of the debugger is:: :file:`pdb.py` can also be invoked as a script to debug other scripts. For example:: - python3 -m pdb myscript.py + python -m pdb myscript.py When invoked as a script, pdb will automatically enter post-mortem debugging if the program being debugged exits abnormally. After post-mortem debugging (or @@ -72,7 +72,7 @@ useful than quitting the debugger upon program's exit. .. versionadded:: 3.7 :file:`pdb.py` now accepts a ``-m`` option that execute modules similar to the way - ``python3 -m`` does. As with a script, the debugger will pause execution just + ``python -m`` does. As with a script, the debugger will pause execution just before the first line of the module. diff --git a/Doc/tutorial/venv.rst b/Doc/tutorial/venv.rst index 05f0e6bbcc1b04f..d1bba098d7d23bf 100644 --- a/Doc/tutorial/venv.rst +++ b/Doc/tutorial/venv.rst @@ -44,7 +44,7 @@ whichever version you want. To create a virtual environment, decide upon a directory where you want to place it, and run the :mod:`venv` module as a script with the directory path:: - python3 -m venv tutorial-env + python -m venv tutorial-env This will create the ``tutorial-env`` directory if it doesn't exist, and also create directories inside it containing a copy of the Python diff --git a/Doc/using/venv-create.inc b/Doc/using/venv-create.inc index d535b254f05698b..43ee6b7807d57e1 100644 --- a/Doc/using/venv-create.inc +++ b/Doc/using/venv-create.inc @@ -1,7 +1,7 @@ Creation of :ref:`virtual environments ` is done by executing the command ``venv``:: - python3 -m venv /path/to/new/virtual/environment + python -m venv /path/to/new/virtual/environment Running this command creates the target directory (creating any parent directories that don't exist already) and places a ``pyvenv.cfg`` file in it From cdb21ba74d933e262bc1696b6ce78b50cb5a4443 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 14 Mar 2023 14:01:35 -0600 Subject: [PATCH 91/97] gh-102660: Handle m_copy Specially for the sys and builtins Modules (gh-102661) It doesn't make sense to use multi-phase init for these modules. Using a per-interpreter "m_copy" (instead of PyModuleDef.m_base.m_copy) makes this work okay. (This came up while working on gh-101660.) Note that we might instead end up disallowing re-load for sys/builtins since they are so special. https://github.com/python/cpython/issues/102660 --- Include/internal/pycore_interp.h | 1 + Python/bltinmodule.c | 3 +++ Python/import.c | 35 ++++++++++++++++++++++++++++---- Python/pystate.c | 1 + Python/sysmodule.c | 8 ++++++++ 5 files changed, 44 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 9efed0a1cf90c27..84303318d218111 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -111,6 +111,7 @@ struct _is { PyObject *dict; /* Stores per-interpreter state */ + PyObject *sysdict_copy; PyObject *builtins_copy; // Initialized to _PyEval_EvalFrameDefault(). _PyFrameEvalFunction eval_frame; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 12ca0ba6c4873cd..a8a34620b9bcdfa 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -3098,6 +3098,9 @@ _PyBuiltin_Init(PyInterpreterState *interp) } Py_DECREF(debug); + /* m_copy of Py_None means it is copied some other way. */ + builtinsmodule.m_base.m_copy = Py_NewRef(Py_None); + return mod; #undef ADD_TO_ALL #undef SETBUILTIN diff --git a/Python/import.c b/Python/import.c index 1bf4199e125aa5d..612fee243bd9e7e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -978,6 +978,16 @@ _PyImport_CheckSubinterpIncompatibleExtensionAllowed(const char *name) return 0; } +static inline int +match_mod_name(PyObject *actual, const char *expected) +{ + if (PyUnicode_CompareWithASCIIString(actual, expected) == 0) { + return 1; + } + assert(!PyErr_Occurred()); + return 0; +} + static int fix_up_extension(PyObject *mod, PyObject *name, PyObject *filename) { @@ -1001,7 +1011,8 @@ fix_up_extension(PyObject *mod, PyObject *name, PyObject *filename) // when the extension module doesn't support sub-interpreters. // XXX Why special-case the main interpreter? if (_Py_IsMainInterpreter(tstate->interp) || def->m_size == -1) { - if (def->m_size == -1) { + /* m_copy of Py_None means it is copied some other way. */ + if (def->m_size == -1 && def->m_base.m_copy != Py_None) { if (def->m_base.m_copy) { /* Somebody already imported the module, likely under a different name. @@ -1055,18 +1066,34 @@ import_find_extension(PyThreadState *tstate, PyObject *name, PyObject *modules = MODULES(tstate->interp); if (def->m_size == -1) { + PyObject *m_copy = def->m_base.m_copy; /* Module does not support repeated initialization */ - if (def->m_base.m_copy == NULL) + if (m_copy == NULL) { return NULL; + } + else if (m_copy == Py_None) { + if (match_mod_name(name, "sys")) { + m_copy = tstate->interp->sysdict_copy; + } + else if (match_mod_name(name, "builtins")) { + m_copy = tstate->interp->builtins_copy; + } + else { + _PyErr_SetString(tstate, PyExc_ImportError, "missing m_copy"); + return NULL; + } + } + /* m_copy of Py_None means it is copied some other way. */ mod = import_add_module(tstate, name); - if (mod == NULL) + if (mod == NULL) { return NULL; + } mdict = PyModule_GetDict(mod); if (mdict == NULL) { Py_DECREF(mod); return NULL; } - if (PyDict_Update(mdict, def->m_base.m_copy)) { + if (PyDict_Update(mdict, m_copy)) { Py_DECREF(mod); return NULL; } diff --git a/Python/pystate.c b/Python/pystate.c index 28606e4f32f71cf..3a2966c54a4c3bb 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -805,6 +805,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) assert(interp->imports.importlib == NULL); assert(interp->imports.import_func == NULL); + Py_CLEAR(interp->sysdict_copy); Py_CLEAR(interp->builtins_copy); Py_CLEAR(interp->dict); #ifdef HAVE_FORK diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 764fb70bae6c384..fc0550266bf1af3 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -3425,12 +3425,20 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) return _PyStatus_ERR("failed to create a module object"); } + /* m_copy of Py_None means it is copied some other way. */ + sysmodule.m_base.m_copy = Py_NewRef(Py_None); + PyObject *sysdict = PyModule_GetDict(sysmod); if (sysdict == NULL) { goto error; } interp->sysdict = Py_NewRef(sysdict); + interp->sysdict_copy = PyDict_Copy(sysdict); + if (interp->sysdict_copy == NULL) { + goto error; + } + if (PyDict_SetItemString(sysdict, "modules", modules) < 0) { goto error; } From d77c48740f5cd17597693bd0d27e32db725fa3a0 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Wed, 15 Mar 2023 05:20:14 +0900 Subject: [PATCH 92/97] gh-102674: Remove _specialization_stats from Lib/opcode.py (#102685) It's not use except in a test, so move it there instead. --- Lib/opcode.py | 8 -------- Lib/test/test__opcode.py | 3 +-- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/Lib/opcode.py b/Lib/opcode.py index 23529d87a09ef94..d4a2d554cf67def 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -377,14 +377,6 @@ def pseudo_op(name, op, real_ops): _specialized_instructions = [ opcode for family in _specializations.values() for opcode in family ] -_specialization_stats = [ - "success", - "failure", - "hit", - "deferred", - "miss", - "deopt", -] _cache_format = { "LOAD_GLOBAL": { diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index db831069c7aeb8b..fb4ab15f7041ed9 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -69,8 +69,7 @@ def test_stack_effect_jump(self): class SpecializationStatsTests(unittest.TestCase): def test_specialization_stats(self): - stat_names = opcode._specialization_stats - + stat_names = ["success", "failure", "hit", "deferred", "miss", "deopt"] specialized_opcodes = [ op.lower() for op in opcode._specializations From a028778d4c813914ae1e6ef3a04bb96dbf92ace6 Mon Sep 17 00:00:00 2001 From: JosephSBoyle <48555120+JosephSBoyle@users.noreply.github.com> Date: Tue, 14 Mar 2023 22:16:45 +0000 Subject: [PATCH 93/97] Rename redundant enum tests so that they run (#102535) --- Lib/test/test_enum.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 2b14590b2c21af5..a11bb441f06e8ee 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -4579,15 +4579,14 @@ class Double(Enum): TWO = 2 self.assertEqual(Double.__doc__, None) - - def test_doc_1(self): + def test_doc_3(self): class Triple(Enum): ONE = 1 TWO = 2 THREE = 3 self.assertEqual(Triple.__doc__, None) - def test_doc_1(self): + def test_doc_4(self): class Quadruple(Enum): ONE = 1 TWO = 2 From 152292b98fd52f8b3db252e5609d6a605a11cb57 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Tue, 14 Mar 2023 22:38:15 +0000 Subject: [PATCH 94/97] gh-101578: mention in what's new in 3.12 that exceptions are now normalized before stored (#102702) --- Doc/whatsnew/3.12.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 9f33dbc808ddc02..217ffec1ee10074 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -982,6 +982,11 @@ Porting to Python 3.12 effects, these side effects are no longer duplicated. (Contributed by Victor Stinner in :gh:`98724`.) +* The interpreter's error indicator is now always normalized. This means + that :c:func:`PyErr_SetObject`, :c:func:`PyErr_SetString` and the other + functions that set the error indicator now normalize the exception + before storing it. (Contributed by Mark Shannon in :gh:`101578`.) + Deprecated ---------- From 0a539b5db312d126ff45dd4aa6a53d40a292c512 Mon Sep 17 00:00:00 2001 From: "Robert Prater (B. Eng)" Date: Tue, 14 Mar 2023 20:03:43 -0400 Subject: [PATCH 95/97] gh-102703: Fix typo in modules tutorial documentation (GH-102707) **Before** This prevents directories with a common name, such as ``string``, unintentionally hiding ... **After** This prevents directories with a common name, such as ``string``, from unintentionally hiding ... --- Doc/tutorial/modules.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index ad70d92994af49b..4daafa49a34d2e6 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -438,7 +438,7 @@ When importing the package, Python searches through the directories on The :file:`__init__.py` files are required to make Python treat directories containing the file as packages. This prevents directories with a common name, -such as ``string``, unintentionally hiding valid modules that occur later +such as ``string``, from unintentionally hiding valid modules that occur later on the module search path. In the simplest case, :file:`__init__.py` can just be an empty file, but it can also execute initialization code for the package or set the ``__all__`` variable, described later. From 5fce813d8e547d6508daa386b67f230105c3a174 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 15 Mar 2023 00:07:30 +0000 Subject: [PATCH 96/97] gh-102519: Avoid failing tests due to inaccessible volumes (GH-102706) --- Lib/test/test_os.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 253e2a23238f12a..42357fef80ec89b 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2683,12 +2683,17 @@ def test_listvolumes(self): def test_listmounts(self): for volume in os.listvolumes(): - mounts = os.listmounts(volume) - self.assertIsInstance(mounts, list) - self.assertSetEqual( - set(mounts), - self.known_mounts & set(mounts), - ) + try: + mounts = os.listmounts(volume) + except OSError as ex: + if support.verbose: + print("Skipping", volume, "because of", ex) + else: + self.assertIsInstance(mounts, list) + self.assertSetEqual( + set(mounts), + self.known_mounts & set(mounts), + ) @unittest.skipUnless(hasattr(os, 'readlink'), 'needs os.readlink()') From e94edab727d07bef851e0e1a345e18a453be863d Mon Sep 17 00:00:00 2001 From: JosephSBoyle <48555120+JosephSBoyle@users.noreply.github.com> Date: Wed, 15 Mar 2023 00:33:19 +0000 Subject: [PATCH 97/97] gh-102560 Add docstrings to asyncio.TaskGroup (#102565) --- Lib/asyncio/taskgroups.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 911419e1769c17a..0fdea3697ece3d6 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -10,7 +10,21 @@ class TaskGroup: + """Asynchronous context manager for managing groups of tasks. + Example use: + + async with asyncio.TaskGroup() as group: + task1 = group.create_task(some_coroutine(...)) + task2 = group.create_task(other_coroutine(...)) + print("Both tasks have completed now.") + + All tasks are awaited when the context manager exits. + + Any exceptions other than `asyncio.CancelledError` raised within + a task will cancel all remaining tasks and wait for them to exit. + The exceptions are then combined and raised as an `ExceptionGroup`. + """ def __init__(self): self._entered = False self._exiting = False @@ -135,6 +149,10 @@ async def __aexit__(self, et, exc, tb): self._errors = None def create_task(self, coro, *, name=None, context=None): + """Create a new task in this group and return it. + + Similar to `asyncio.create_task`. + """ if not self._entered: raise RuntimeError(f"TaskGroup {self!r} has not been entered") if self._exiting and not self._tasks: