From ffe47cb623999db05959ec4b5168d1c87a1e40ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marta=20G=C3=B3mez=20Mac=C3=ADas?= Date: Sun, 21 May 2023 18:07:28 +0200 Subject: [PATCH 01/47] gh-104719: Restore Tokenize module constants (#104722) --- Lib/tokenize.py | 101 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/Lib/tokenize.py b/Lib/tokenize.py index bfe40c627fde57..cef2773feac24b 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -56,6 +56,107 @@ def exact_type(self): else: return self.type +def group(*choices): return '(' + '|'.join(choices) + ')' +def any(*choices): return group(*choices) + '*' +def maybe(*choices): return group(*choices) + '?' + +# Note: we use unicode matching for names ("\w") but ascii matching for +# number literals. +Whitespace = r'[ \f\t]*' +Comment = r'#[^\r\n]*' +Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) +Name = r'\w+' + +Hexnumber = r'0[xX](?:_?[0-9a-fA-F])+' +Binnumber = r'0[bB](?:_?[01])+' +Octnumber = r'0[oO](?:_?[0-7])+' +Decnumber = r'(?:0(?:_?0)*|[1-9](?:_?[0-9])*)' +Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber) +Exponent = r'[eE][-+]?[0-9](?:_?[0-9])*' +Pointfloat = group(r'[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?', + r'\.[0-9](?:_?[0-9])*') + maybe(Exponent) +Expfloat = r'[0-9](?:_?[0-9])*' + Exponent +Floatnumber = group(Pointfloat, Expfloat) +Imagnumber = group(r'[0-9](?:_?[0-9])*[jJ]', Floatnumber + r'[jJ]') +Number = group(Imagnumber, Floatnumber, Intnumber) + +# Return the empty string, plus all of the valid string prefixes. +def _all_string_prefixes(): + # The valid string prefixes. Only contain the lower case versions, + # and don't contain any permutations (include 'fr', but not + # 'rf'). The various permutations will be generated. + _valid_string_prefixes = ['b', 'r', 'u', 'f', 'br', 'fr'] + # if we add binary f-strings, add: ['fb', 'fbr'] + result = {''} + for prefix in _valid_string_prefixes: + for t in _itertools.permutations(prefix): + # create a list with upper and lower versions of each + # character + for u in _itertools.product(*[(c, c.upper()) for c in t]): + result.add(''.join(u)) + return result + +@functools.lru_cache +def _compile(expr): + return re.compile(expr, re.UNICODE) + +# Note that since _all_string_prefixes includes the empty string, +# StringPrefix can be the empty string (making it optional). +StringPrefix = group(*_all_string_prefixes()) + +# Tail end of ' string. +Single = r"[^'\\]*(?:\\.[^'\\]*)*'" +# Tail end of " string. +Double = r'[^"\\]*(?:\\.[^"\\]*)*"' +# Tail end of ''' string. +Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''" +# Tail end of """ string. +Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""' +Triple = group(StringPrefix + "'''", StringPrefix + '"""') +# Single-line ' or " string. +String = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'", + StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"') + +# Sorting in reverse order puts the long operators before their prefixes. +# Otherwise if = came before ==, == would get recognized as two instances +# of =. +Special = group(*map(re.escape, sorted(EXACT_TOKEN_TYPES, reverse=True))) +Funny = group(r'\r?\n', Special) + +PlainToken = group(Number, Funny, String, Name) +Token = Ignore + PlainToken + +# First (or only) line of ' or " string. +ContStr = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" + + group("'", r'\\\r?\n'), + StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' + + group('"', r'\\\r?\n')) +PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple) +PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) + +# For a given string prefix plus quotes, endpats maps it to a regex +# to match the remainder of that string. _prefix can be empty, for +# a normal single or triple quoted string (with no prefix). +endpats = {} +for _prefix in _all_string_prefixes(): + endpats[_prefix + "'"] = Single + endpats[_prefix + '"'] = Double + endpats[_prefix + "'''"] = Single3 + endpats[_prefix + '"""'] = Double3 +del _prefix + +# A set of all of the single and triple quoted string prefixes, +# including the opening quotes. +single_quoted = set() +triple_quoted = set() +for t in _all_string_prefixes(): + for u in (t + '"', t + "'"): + single_quoted.add(u) + for u in (t + '"""', t + "'''"): + triple_quoted.add(u) +del t, u + +tabsize = 8 class TokenError(Exception): pass From 0c5e79bcdf99a6a07c65b0ea6d7701b6e041481c Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 21 May 2023 14:42:26 -0400 Subject: [PATCH 02/47] gh-104719: IDLE - delete useless monkeypatch of tokenize (#104726) --- Lib/idlelib/editor.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 21402ad7139173..df36be8766016f 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -1643,19 +1643,13 @@ def tokeneater(self, type, token, start, end, line, self.finished = 1 def run(self): - save_tabsize = tokenize.tabsize - tokenize.tabsize = self.tabwidth try: - try: - tokens = tokenize.generate_tokens(self.readline) - for token in tokens: - self.tokeneater(*token) - except (tokenize.TokenError, SyntaxError): - # since we cut off the tokenizer early, we can trigger - # spurious errors - pass - finally: - tokenize.tabsize = save_tabsize + tokens = tokenize.generate_tokens(self.readline) + for token in tokens: + self.tokeneater(*token) + except (tokenize.TokenError, SyntaxError): + # Stopping the tokenizer early can trigger spurious errors. + pass return self.blkopenline, self.indentedline ### end autoindent code ### From 5841fbc1a2aa5dd9e1c3d8d369f2bba8a99b285b Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 21 May 2023 22:45:44 +0300 Subject: [PATCH 03/47] gh-103857: Document utcnow and utcfromtimestamp deprecations in What's New (#104542) Co-authored-by: Paul Ganssle <1377457+pganssle@users.noreply.github.com> --- Doc/whatsnew/3.12.rst | 10 ++++++++++ Modules/_datetimemodule.c | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 8a0a59ba7301e1..14f03ef755c734 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -774,6 +774,16 @@ Deprecated ``int``, convert to int explicitly with ``~int(x)``. (Contributed by Tim Hoffmann in :gh:`103487`.) +* :class:`datetime.datetime`'s + :meth:`~datetime.datetime.utcnow` and + :meth:`~datetime.datetime.utcfromtimestamp` are deprecated and will be + removed in a future version. Instead, use timezone-aware objects to represent + datetimes in UTC: respectively, call + :meth:`~datetime.datetime.now` and + :meth:`~datetime.datetime.fromtimestamp` with the *tz* parameter set to + :attr:`datetime.UTC`. + (Contributed by Paul Ganssle in :gh:`103857`.) + Pending Removal in Python 3.13 ------------------------------ diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 8b417bdd0fb5b6..19e11780ec6e19 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -5190,7 +5190,7 @@ datetime_utcfromtimestamp(PyObject *cls, PyObject *args) if (PyErr_WarnEx(PyExc_DeprecationWarning, "datetime.utcfromtimestamp() is deprecated and scheduled for removal " "in a future version. Use timezone-aware objects to represent " - "datetimes in UTC: datetime.now(datetime.UTC).", 1)) + "datetimes in UTC: datetime.fromtimestamp(timestamp, datetime.UTC).", 1)) { return NULL; } From c482e9a4e6f9c0b88ea248c4a0e0aa97223b5233 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 21 May 2023 22:23:14 +0200 Subject: [PATCH 04/47] gh-104050: Annotate Argument Clinic return converters (#104706) Co-authored-by: Alex Waygood --- Tools/clinic/clinic.py | 105 +++++++++++++++++++++++++++++------------ 1 file changed, 75 insertions(+), 30 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 863bd66bef754d..a1e8947529a0d4 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -1966,15 +1966,6 @@ def dump(self): extensions['py'] = PythonLanguage -# maps strings to callables. -# these callables must be of the form: -# def foo(*, ...) -# The callable may have any number of keyword-only parameters. -# The callable must return a CConverter object. -# The callable should not call builtins.print. -return_converters = {} - - def file_changed(filename: str, new_contents: str) -> bool: """Return true if file contents changed (meaning we must update it)""" try: @@ -3005,6 +2996,16 @@ def parser_name(self): # note however that they will never be called with keyword-only parameters. legacy_converters: ConverterDict = {} +# maps strings to callables. +# these callables must be of the form: +# def foo(*, ...) +# The callable may have any number of keyword-only parameters. +# The callable must return a CReturnConverter object. +# The callable should not call builtins.print. +ReturnConverterType = Callable[..., "CReturnConverter"] +ReturnConverterDict = dict[str, ReturnConverterType] +return_converters: ReturnConverterDict = {} + TypeSet = set[bltns.type[Any]] @@ -3966,8 +3967,10 @@ def set_template_dict(self, template_dict): template_dict['base_type_ptr'] = type_ptr - -def add_c_return_converter(f, name=None): +def add_c_return_converter( + f: ReturnConverterType, + name: str | None = None +) -> ReturnConverterType: if not name: name = f.__name__ if not name.endswith('_return_converter'): @@ -3978,9 +3981,15 @@ def add_c_return_converter(f, name=None): class CReturnConverterAutoRegister(type): - def __init__(cls, name, bases, classdict): + def __init__( + cls: ReturnConverterType, + name: str, + bases: tuple[type, ...], + classdict: dict[str, Any] + ) -> None: add_c_return_converter(cls) + class CReturnConverter(metaclass=CReturnConverterAutoRegister): # The C type to use for this variable. @@ -3992,7 +4001,12 @@ class CReturnConverter(metaclass=CReturnConverterAutoRegister): # Or the magic value "unspecified" if there is no default. default: object = None - def __init__(self, *, py_default=None, **kwargs): + def __init__( + self, + *, + py_default: str | None = None, + **kwargs + ) -> None: self.py_default = py_default try: self.return_converter_init(**kwargs) @@ -4000,11 +4014,10 @@ def __init__(self, *, py_default=None, **kwargs): s = ', '.join(name + '=' + repr(value) for name, value in kwargs.items()) sys.exit(self.__class__.__name__ + '(' + s + ')\n' + str(e)) - def return_converter_init(self): - pass + def return_converter_init(self) -> None: ... - def declare(self, data): - line = [] + def declare(self, data: CRenderData) -> None: + line: list[str] = [] add = line.append add(self.type) if not self.type.endswith('*'): @@ -4013,50 +4026,70 @@ def declare(self, data): data.declarations.append(''.join(line)) data.return_value = data.converter_retval - def err_occurred_if(self, expr, data): + def err_occurred_if( + self, + expr: str, + data: CRenderData + ) -> None: line = f'if (({expr}) && PyErr_Occurred()) {{\n goto exit;\n}}\n' data.return_conversion.append(line) - def err_occurred_if_null_pointer(self, variable, data): + def err_occurred_if_null_pointer( + self, + variable: str, + data: CRenderData + ) -> None: line = f'if ({variable} == NULL) {{\n goto exit;\n}}\n' data.return_conversion.append(line) - def render(self, function, data): - """ - function is a clinic.Function instance. - data is a CRenderData instance. - """ - pass + def render( + self, + function: Function, + data: CRenderData + ) -> None: ... + add_c_return_converter(CReturnConverter, 'object') + class bool_return_converter(CReturnConverter): type = 'int' - def render(self, function, data): + def render( + self, + function: Function, + data: CRenderData + ) -> None: self.declare(data) self.err_occurred_if(f"{data.converter_retval} == -1", data) data.return_conversion.append( f'return_value = PyBool_FromLong((long){data.converter_retval});\n' ) + class long_return_converter(CReturnConverter): type = 'long' conversion_fn = 'PyLong_FromLong' cast = '' unsigned_cast = '' - def render(self, function, data): + def render( + self, + function: Function, + data: CRenderData + ) -> None: self.declare(data) self.err_occurred_if(f"{data.converter_retval} == {self.unsigned_cast}-1", data) data.return_conversion.append( f'return_value = {self.conversion_fn}({self.cast}{data.converter_retval});\n' ) + class int_return_converter(long_return_converter): type = 'int' cast = '(long)' + class init_return_converter(long_return_converter): """ Special return converter for __init__ functions. @@ -4064,23 +4097,30 @@ class init_return_converter(long_return_converter): type = 'int' cast = '(long)' - def render(self, function, data): - pass + def render( + self, + function: Function, + data: CRenderData + ) -> None: ... + class unsigned_long_return_converter(long_return_converter): type = 'unsigned long' conversion_fn = 'PyLong_FromUnsignedLong' unsigned_cast = '(unsigned long)' + class unsigned_int_return_converter(unsigned_long_return_converter): type = 'unsigned int' cast = '(unsigned long)' unsigned_cast = '(unsigned int)' + class Py_ssize_t_return_converter(long_return_converter): type = 'Py_ssize_t' conversion_fn = 'PyLong_FromSsize_t' + class size_t_return_converter(long_return_converter): type = 'size_t' conversion_fn = 'PyLong_FromSize_t' @@ -4091,13 +4131,18 @@ class double_return_converter(CReturnConverter): type = 'double' cast = '' - def render(self, function, data): + def render( + self, + function: Function, + data: CRenderData + ) -> None: self.declare(data) self.err_occurred_if(f"{data.converter_retval} == -1.0", data) data.return_conversion.append( f'return_value = PyFloat_FromDouble({self.cast}{data.converter_retval});\n' ) + class float_return_converter(double_return_converter): type = 'float' cast = '(double)' From bf6dd8f5fd98a1833646eb22269cad076836ebba Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 21 May 2023 22:49:34 +0200 Subject: [PATCH 05/47] gh-104050: Add basic type hints to Argument Clinic clinic class (#104705) Co-authored-by: Alex Waygood --- Tools/clinic/clinic.py | 55 +++++++++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index a1e8947529a0d4..513586ee019115 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -1989,6 +1989,11 @@ def write_file(filename: str, new_contents: str): raise +ClassDict = dict[str, "Class"] +DestinationDict = dict[str, Destination] +ModuleDict = dict[str, "Module"] +ParserDict = dict[str, "DSLParser"] + clinic = None class Clinic: @@ -2035,23 +2040,30 @@ class Clinic: """ - def __init__(self, language, printer=None, *, verify=True, filename=None): + def __init__( + self, + language: CLanguage, + printer: BlockPrinter | None = None, + *, + verify: bool = True, + filename: str | None = None + ) -> None: # maps strings to Parser objects. # (instantiated from the "parsers" global.) - self.parsers = {} - self.language = language + self.parsers: ParserDict = {} + self.language: CLanguage = language if printer: fail("Custom printers are broken right now") self.printer = printer or BlockPrinter(language) self.verify = verify self.filename = filename - self.modules = {} - self.classes = {} - self.functions = [] + self.modules: ModuleDict = {} + self.classes: ClassDict = {} + self.functions: list[Function] = [] self.line_prefix = self.line_suffix = '' - self.destinations = {} + self.destinations: DestinationDict = {} self.add_destination("block", "buffer") self.add_destination("suppress", "suppress") self.add_destination("buffer", "buffer") @@ -2072,10 +2084,13 @@ def __init__(self, language, printer=None, *, verify=True, filename=None): 'impl_definition': d('block'), } - self.destination_buffers_stack = [] - self.ifndef_symbols = set() + DestBufferType = dict[str, Callable[..., Any]] + DestBufferList = list[DestBufferType] + + self.destination_buffers_stack: DestBufferList = [] + self.ifndef_symbols: set[str] = set() - self.presets = {} + self.presets: dict[str, dict[Any, Any]] = {} preset = None for line in self.presets_text.strip().split('\n'): line = line.strip() @@ -2103,18 +2118,27 @@ def __init__(self, language, printer=None, *, verify=True, filename=None): global clinic clinic = self - def add_destination(self, name, type, *args): + def add_destination( + self, + name: str, + type: str, + *args + ) -> None: if name in self.destinations: fail("Destination already exists: " + repr(name)) self.destinations[name] = Destination(name, type, self, *args) - def get_destination(self, name): + def get_destination(self, name: str) -> Destination: d = self.destinations.get(name) if not d: fail("Destination does not exist: " + repr(name)) return d - def get_destination_buffer(self, name, item=0): + def get_destination_buffer( + self, + name: str, + item: int = 0 + ): d = self.get_destination(name) return d.buffers[item] @@ -2240,6 +2264,7 @@ def parse_file( if not find_start_re.search(raw): return + assert isinstance(language, CLanguage) clinic = Clinic(language, verify=verify, filename=filename) src_out, clinic_out = clinic.parse(raw) @@ -2275,8 +2300,6 @@ def parse(self, block: Block) -> None: block.output = s.getvalue() -ModuleDict = dict[str, "Module"] - class Module: def __init__( self, @@ -2294,8 +2317,6 @@ def __repr__(self) -> str: return "" -ClassDict = dict[str, "Class"] - class Class: def __init__( self, From 6ba8406cb6e656e47e908f8c7354e07ed0f2d774 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 21 May 2023 23:24:26 +0200 Subject: [PATCH 06/47] gh-104050: Add more type annotations to Argument Clinic (#104631) Annotate methods of the following classes: - class Function - class Parameter - class LandMine --- Tools/clinic/clinic.py | 67 +++++++++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 23 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 513586ee019115..1d7b778da70990 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -2416,6 +2416,9 @@ def __repr__(self) -> str: INVALID, CALLABLE, STATIC_METHOD, CLASS_METHOD, METHOD_INIT, METHOD_NEW """.replace(",", "").strip().split() +ParamDict = dict[str, "Parameter"] +ReturnConverterType = Callable[..., "CReturnConverter"] + class Function: """ Mutable duck type for inspect.Function. @@ -2428,12 +2431,22 @@ class Function: (not docstring) or ((not docstring[0].isspace()) and (docstring.rstrip() == docstring)) """ - def __init__(self, parameters=None, *, name, - module, cls=None, c_basename=None, - full_name=None, - return_converter, return_annotation=inspect.Signature.empty, - docstring=None, kind=CALLABLE, coexist=False, - docstring_only=False): + def __init__( + self, + parameters: ParamDict | None = None, + *, + name: str, + module: Module, + cls: Class | None = None, + c_basename: str | None = None, + full_name: str | None = None, + return_converter: ReturnConverterType, + return_annotation = inspect.Signature.empty, + docstring: str | None = None, + kind: str = CALLABLE, + coexist: bool = False, + docstring_only: bool = False + ) -> None: self.parameters = parameters or {} self.return_annotation = return_annotation self.name = name @@ -2467,7 +2480,7 @@ def render_parameters(self): return self.__render_parameters__ @property - def methoddef_flags(self): + def methoddef_flags(self) -> str | None: if self.kind in (METHOD_INIT, METHOD_NEW): return None flags = [] @@ -2481,10 +2494,10 @@ def methoddef_flags(self): flags.append('METH_COEXIST') return '|'.join(flags) - def __repr__(self): + def __repr__(self) -> str: return '' - def copy(self, **overrides): + def copy(self, **overrides) -> "Function": kwargs = { 'name': self.name, 'module': self.module, 'parameters': self.parameters, 'cls': self.cls, 'c_basename': self.c_basename, @@ -2507,9 +2520,18 @@ class Parameter: Mutable duck type of inspect.Parameter. """ - def __init__(self, name, kind, *, default=inspect.Parameter.empty, - function, converter, annotation=inspect.Parameter.empty, - docstring=None, group=0): + def __init__( + self, + name: str, + kind: str, + *, + default = inspect.Parameter.empty, + function: Function, + converter: "CConverter", + annotation = inspect.Parameter.empty, + docstring: str | None = None, + group: int = 0 + ) -> None: self.name = name self.kind = kind self.default = default @@ -2519,22 +2541,22 @@ def __init__(self, name, kind, *, default=inspect.Parameter.empty, self.docstring = docstring or '' self.group = group - def __repr__(self): + def __repr__(self) -> str: return '' - def is_keyword_only(self): + def is_keyword_only(self) -> bool: return self.kind == inspect.Parameter.KEYWORD_ONLY - def is_positional_only(self): + def is_positional_only(self) -> bool: return self.kind == inspect.Parameter.POSITIONAL_ONLY - def is_vararg(self): + def is_vararg(self) -> bool: return self.kind == inspect.Parameter.VAR_POSITIONAL - def is_optional(self): + def is_optional(self) -> bool: return not self.is_vararg() and (self.default is not unspecified) - def copy(self, **overrides): + def copy(self, **overrides) -> "Parameter": kwargs = { 'name': self.name, 'kind': self.kind, 'default':self.default, 'function': self.function, 'converter': self.converter, 'annotation': self.annotation, @@ -2547,7 +2569,7 @@ def copy(self, **overrides): kwargs['converter'] = converter return Parameter(**kwargs) - def get_displayname(self, i): + def get_displayname(self, i: int) -> str: if i == 0: return '"argument"' if not self.is_positional_only(): @@ -2558,13 +2580,13 @@ def get_displayname(self, i): class LandMine: # try to access any - def __init__(self, message): + def __init__(self, message: str) -> None: self.__message__ = message - def __repr__(self): + def __repr__(self) -> str: return '" - def __getattribute__(self, name): + def __getattribute__(self, name: str): if name in ('__repr__', '__message__'): return super().__getattribute__(name) # raise RuntimeError(repr(name)) @@ -3023,7 +3045,6 @@ def parser_name(self): # The callable may have any number of keyword-only parameters. # The callable must return a CReturnConverter object. # The callable should not call builtins.print. -ReturnConverterType = Callable[..., "CReturnConverter"] ReturnConverterDict = dict[str, ReturnConverterType] return_converters: ReturnConverterDict = {} From f3466bc04008660c4a5c3ed6f70144f138ae2e7f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 22 May 2023 00:32:39 +0300 Subject: [PATCH 07/47] gh-98836: Extend PyUnicode_FromFormat() (GH-98838) * Support for conversion specifiers o (octal) and X (uppercase hexadecimal). * Support for length modifiers j (intmax_t) and t (ptrdiff_t). * Length modifiers are now applied to all integer conversions. * Support for wchar_t C strings (%ls and %lV). * Support for variable width and precision (*). * Support for flag - (left alignment). --- Doc/c-api/unicode.rst | 228 +++++++----- Doc/whatsnew/3.12.rst | 6 + Lib/test/test_capi/test_unicode.py | 217 ++++++++---- ...2-10-29-10-13-20.gh-issue-98836.Cy5h_z.rst | 4 + Modules/_ssl.c | 5 +- Modules/_testcapi/unicode.c | 63 +++- Modules/selectmodule.c | 5 +- Modules/socketmodule.c | 8 +- Objects/unicodeobject.c | 326 ++++++++++++------ Parser/tokenizer.c | 11 +- 10 files changed, 585 insertions(+), 288 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2022-10-29-10-13-20.gh-issue-98836.Cy5h_z.rst diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index ab3a2e274d9395..6771f378bfbc31 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -394,98 +394,149 @@ APIs: arguments, calculate the size of the resulting Python Unicode string and return a string with the values formatted into it. The variable arguments must be C types and must correspond exactly to the format characters in the *format* - ASCII-encoded string. The following format characters are allowed: - - .. % This should be exactly the same as the table in PyErr_Format. - - .. tabularcolumns:: |l|l|L| - - +-------------------+---------------------+----------------------------------+ - | Format Characters | Type | Comment | - +===================+=====================+==================================+ - | :attr:`%%` | *n/a* | The literal % character. | - +-------------------+---------------------+----------------------------------+ - | :attr:`%c` | int | A single character, | - | | | represented as a C int. | - +-------------------+---------------------+----------------------------------+ - | :attr:`%d` | int | Equivalent to | - | | | ``printf("%d")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%u` | unsigned int | Equivalent to | - | | | ``printf("%u")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%ld` | long | Equivalent to | - | | | ``printf("%ld")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%li` | long | Equivalent to | - | | | ``printf("%li")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%lu` | unsigned long | Equivalent to | - | | | ``printf("%lu")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%lld` | long long | Equivalent to | - | | | ``printf("%lld")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%lli` | long long | Equivalent to | - | | | ``printf("%lli")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%llu` | unsigned long long | Equivalent to | - | | | ``printf("%llu")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%zd` | :c:type:`\ | Equivalent to | - | | Py_ssize_t` | ``printf("%zd")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%zi` | :c:type:`\ | Equivalent to | - | | Py_ssize_t` | ``printf("%zi")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%zu` | size_t | Equivalent to | - | | | ``printf("%zu")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%i` | int | Equivalent to | - | | | ``printf("%i")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%x` | int | Equivalent to | - | | | ``printf("%x")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%s` | const char\* | A null-terminated C character | - | | | array. | - +-------------------+---------------------+----------------------------------+ - | :attr:`%p` | const void\* | The hex representation of a C | - | | | pointer. Mostly equivalent to | - | | | ``printf("%p")`` except that | - | | | it is guaranteed to start with | - | | | the literal ``0x`` regardless | - | | | of what the platform's | - | | | ``printf`` yields. | - +-------------------+---------------------+----------------------------------+ - | :attr:`%A` | PyObject\* | The result of calling | - | | | :func:`ascii`. | - +-------------------+---------------------+----------------------------------+ - | :attr:`%U` | PyObject\* | A Unicode object. | - +-------------------+---------------------+----------------------------------+ - | :attr:`%V` | PyObject\*, | A Unicode object (which may be | - | | const char\* | ``NULL``) and a null-terminated | - | | | C character array as a second | - | | | parameter (which will be used, | - | | | if the first parameter is | - | | | ``NULL``). | - +-------------------+---------------------+----------------------------------+ - | :attr:`%S` | PyObject\* | The result of calling | - | | | :c:func:`PyObject_Str`. | - +-------------------+---------------------+----------------------------------+ - | :attr:`%R` | PyObject\* | The result of calling | - | | | :c:func:`PyObject_Repr`. | - +-------------------+---------------------+----------------------------------+ + ASCII-encoded string. + + A conversion specifier contains two or more characters and has the following + components, which must occur in this order: + + #. The ``'%'`` character, which marks the start of the specifier. + + #. Conversion flags (optional), which affect the result of some conversion + types. + + #. Minimum field width (optional). + If specified as an ``'*'`` (asterisk), the actual width is given in the + next argument, which must be of type :c:expr:`int`, and the object to + convert comes after the minimum field width and optional precision. + + #. Precision (optional), given as a ``'.'`` (dot) followed by the precision. + If specified as ``'*'`` (an asterisk), the actual precision is given in + the next argument, which must be of type :c:expr:`int`, and the value to + convert comes after the precision. + + #. Length modifier (optional). + + #. Conversion type. + + The conversion flag characters are: + + .. tabularcolumns:: |l|L| + + +-------+-------------------------------------------------------------+ + | Flag | Meaning | + +=======+=============================================================+ + | ``0`` | The conversion will be zero padded for numeric values. | + +-------+-------------------------------------------------------------+ + | ``-`` | The converted value is left adjusted (overrides the ``0`` | + | | flag if both are given). | + +-------+-------------------------------------------------------------+ + + The length modifiers for following integer conversions (``d``, ``i``, + ``o``, ``u``, ``x``, or ``X``) specify the type of the argument + (:c:expr:`int` by default): + + .. tabularcolumns:: |l|L| + + +----------+-----------------------------------------------------+ + | Modifier | Types | + +==========+=====================================================+ + | ``l`` | :c:expr:`long` or :c:expr:`unsigned long` | + +----------+-----------------------------------------------------+ + | ``ll`` | :c:expr:`long long` or :c:expr:`unsigned long long` | + +----------+-----------------------------------------------------+ + | ``j`` | :c:expr:`intmax_t` or :c:expr:`uintmax_t` | + +----------+-----------------------------------------------------+ + | ``z`` | :c:expr:`size_t` or :c:expr:`ssize_t` | + +----------+-----------------------------------------------------+ + | ``t`` | :c:expr:`ptrdiff_t` | + +----------+-----------------------------------------------------+ + + The length modifier ``l`` for following conversions ``s`` or ``V`` specify + that the type of the argument is :c:expr:`const wchar_t*`. + + The conversion specifiers are: + + .. list-table:: + :widths: auto + :header-rows: 1 + + * - Conversion Specifier + - Type + - Comment + + * - ``%`` + - *n/a* + - The literal ``%`` character. + + * - ``d``, ``i`` + - Specified by the length modifier + - The decimal representation of a signed C integer. + + * - ``u`` + - Specified by the length modifier + - The decimal representation of an unsigned C integer. + + * - ``o`` + - Specified by the length modifier + - The octal representation of an unsigned C integer. + + * - ``x`` + - Specified by the length modifier + - The hexadecimal representation of an unsigned C integer (lowercase). + + * - ``X`` + - Specified by the length modifier + - The hexadecimal representation of an unsigned C integer (uppercase). + + * - ``c`` + - :c:expr:`int` + - A single character. + + * - ``s`` + - :c:expr:`const char*` or :c:expr:`const wchar_t*` + - A null-terminated C character array. + + * - ``p`` + - :c:expr:`const void*` + - The hex representation of a C pointer. + Mostly equivalent to ``printf("%p")`` except that it is guaranteed to + start with the literal ``0x`` regardless of what the platform's + ``printf`` yields. + + * - ``A`` + - :c:expr:`PyObject*` + - The result of calling :func:`ascii`. + + * - ``U`` + - :c:expr:`PyObject*` + - A Unicode object. + + * - ``V`` + - :c:expr:`PyObject*`, :c:expr:`const char*` or :c:expr:`const wchar_t*` + - A Unicode object (which may be ``NULL``) and a null-terminated + C character array as a second parameter (which will be used, + if the first parameter is ``NULL``). + + * - ``S`` + - :c:expr:`PyObject*` + - The result of calling :c:func:`PyObject_Str`. + + * - ``R`` + - :c:expr:`PyObject*` + - The result of calling :c:func:`PyObject_Repr`. .. note:: The width formatter unit is number of characters rather than bytes. - The precision formatter unit is number of bytes for ``"%s"`` and + The precision formatter unit is number of bytes or :c:expr:`wchar_t` + items (if the length modifier ``l`` is used) for ``"%s"`` and ``"%V"`` (if the ``PyObject*`` argument is ``NULL``), and a number of characters for ``"%A"``, ``"%U"``, ``"%S"``, ``"%R"`` and ``"%V"`` (if the ``PyObject*`` argument is not ``NULL``). - .. [1] For integer specifiers (d, u, ld, li, lu, lld, lli, llu, zd, zi, - zu, i, x): the 0-conversion flag has effect even when a precision is given. + .. note:: + Unlike to C :c:func:`printf` the ``0`` flag has effect even when + a precision is given for integer conversions (``d``, ``i``, ``u``, ``o``, + ``x``, or ``X``). .. versionchanged:: 3.2 Support for ``"%lld"`` and ``"%llu"`` added. @@ -498,6 +549,13 @@ APIs: ``"%V"``, ``"%S"``, ``"%R"`` added. .. versionchanged:: 3.12 + Support for conversion specifiers ``o`` and ``X``. + Support for length modifiers ``j`` and ``t``. + Length modifiers are now applied to all integer conversions. + Length modifier ``l`` is now applied to conversion specifiers ``s`` and ``V``. + Support for variable width and precision ``*``. + Support for flag ``-``. + An unrecognized format character now sets a :exc:`SystemError`. In previous versions it caused all the rest of the format string to be copied as-is to the result string, and any extra arguments discarded. diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 14f03ef755c734..caf21078b9bd59 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1402,6 +1402,12 @@ Porting to Python 3.12 :py:meth:`~class.__subclasses__` (using :c:func:`PyObject_CallMethod`, for example). +* Add support of more formatting options (left aligning, octals, uppercase + hexadecimals, ``intmax_t``, ``ptrdiff_t``, ``wchar_t`` C + strings, variable width and precision) in :c:func:`PyUnicode_FromFormat` and + :c:func:`PyUnicode_FromFormatV`. + (Contributed by Serhiy Storchaka in :gh:`98836`.) + * An unrecognized format character in :c:func:`PyUnicode_FromFormat` and :c:func:`PyUnicode_FromFormatV` now sets a :exc:`SystemError`. In previous versions it caused all the rest of the format string to be diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index 00807d968a7c43..9c7662065689ea 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -319,12 +319,17 @@ def test_fromobject(self): def test_from_format(self): """Test PyUnicode_FromFormat()""" + # Length modifiers "j" and "t" are not tested here because ctypes does + # not expose types for intmax_t and ptrdiff_t. + # _testcapi.test_string_from_format() has a wider coverage of all + # formats. import_helper.import_module('ctypes') from ctypes import ( c_char_p, pythonapi, py_object, sizeof, c_int, c_long, c_longlong, c_ssize_t, - c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p) + c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p, + sizeof, c_wchar, c_wchar_p) name = "PyUnicode_FromFormat" _PyUnicode_FromFormat = getattr(pythonapi, name) _PyUnicode_FromFormat.argtypes = (c_char_p,) @@ -449,37 +454,28 @@ def check_format(expected, format, *args): check_format("repr= 12", b'repr=%5.2V', None, b'123') - # test integer formats (%i, %d, %u) + # test integer formats (%i, %d, %u, %o, %x, %X) check_format('010', b'%03i', c_int(10)) check_format('0010', b'%0.4i', c_int(10)) - check_format('-123', - b'%i', c_int(-123)) - check_format('-123', - b'%li', c_long(-123)) - check_format('-123', - b'%lli', c_longlong(-123)) - check_format('-123', - b'%zi', c_ssize_t(-123)) - - check_format('-123', - b'%d', c_int(-123)) - check_format('-123', - b'%ld', c_long(-123)) - check_format('-123', - b'%lld', c_longlong(-123)) - check_format('-123', - b'%zd', c_ssize_t(-123)) - - check_format('123', - b'%u', c_uint(123)) - check_format('123', - b'%lu', c_ulong(123)) - check_format('123', - b'%llu', c_ulonglong(123)) - check_format('123', - b'%zu', c_size_t(123)) + for conv, signed, value, expected in [ + (b'i', True, -123, '-123'), + (b'd', True, -123, '-123'), + (b'u', False, 123, '123'), + (b'o', False, 0o123, '123'), + (b'x', False, 0xabc, 'abc'), + (b'X', False, 0xabc, 'ABC'), + ]: + for mod, ctype in [ + (b'', c_int if signed else c_uint), + (b'l', c_long if signed else c_ulong), + (b'll', c_longlong if signed else c_ulonglong), + (b'z', c_ssize_t if signed else c_size_t), + ]: + with self.subTest(format=b'%' + mod + conv): + check_format(expected, + b'%' + mod + conv, ctype(value)) # test long output min_longlong = -(2 ** (8 * sizeof(c_longlong) - 1)) @@ -494,40 +490,144 @@ def check_format(expected, format, *args): PyUnicode_FromFormat(b'%p', c_void_p(-1)) # test padding (width and/or precision) - check_format('123'.rjust(10, '0'), - b'%010i', c_int(123)) - check_format('123'.rjust(100), - b'%100i', c_int(123)) - check_format('123'.rjust(100, '0'), - b'%.100i', c_int(123)) - check_format('123'.rjust(80, '0').rjust(100), - b'%100.80i', c_int(123)) - - check_format('123'.rjust(10, '0'), - b'%010u', c_uint(123)) - check_format('123'.rjust(100), - b'%100u', c_uint(123)) - check_format('123'.rjust(100, '0'), - b'%.100u', c_uint(123)) - check_format('123'.rjust(80, '0').rjust(100), - b'%100.80u', c_uint(123)) - - check_format('123'.rjust(10, '0'), - b'%010x', c_int(0x123)) - check_format('123'.rjust(100), - b'%100x', c_int(0x123)) - check_format('123'.rjust(100, '0'), - b'%.100x', c_int(0x123)) - check_format('123'.rjust(80, '0').rjust(100), - b'%100.80x', c_int(0x123)) + check_format('123', b'%2i', c_int(123)) + check_format(' 123', b'%10i', c_int(123)) + check_format('0000000123', b'%010i', c_int(123)) + check_format('123 ', b'%-10i', c_int(123)) + check_format('123 ', b'%-010i', c_int(123)) + check_format('123', b'%.2i', c_int(123)) + check_format('0000123', b'%.7i', c_int(123)) + check_format(' 123', b'%10.2i', c_int(123)) + check_format(' 0000123', b'%10.7i', c_int(123)) + check_format('0000000123', b'%010.7i', c_int(123)) + check_format('0000123 ', b'%-10.7i', c_int(123)) + check_format('0000123 ', b'%-010.7i', c_int(123)) + + check_format('-123', b'%2i', c_int(-123)) + check_format(' -123', b'%10i', c_int(-123)) + check_format('-000000123', b'%010i', c_int(-123)) + check_format('-123 ', b'%-10i', c_int(-123)) + check_format('-123 ', b'%-010i', c_int(-123)) + check_format('-123', b'%.2i', c_int(-123)) + check_format('-0000123', b'%.7i', c_int(-123)) + check_format(' -123', b'%10.2i', c_int(-123)) + check_format(' -0000123', b'%10.7i', c_int(-123)) + check_format('-000000123', b'%010.7i', c_int(-123)) + check_format('-0000123 ', b'%-10.7i', c_int(-123)) + check_format('-0000123 ', b'%-010.7i', c_int(-123)) + + check_format('123', b'%2u', c_uint(123)) + check_format(' 123', b'%10u', c_uint(123)) + check_format('0000000123', b'%010u', c_uint(123)) + check_format('123 ', b'%-10u', c_uint(123)) + check_format('123 ', b'%-010u', c_uint(123)) + check_format('123', b'%.2u', c_uint(123)) + check_format('0000123', b'%.7u', c_uint(123)) + check_format(' 123', b'%10.2u', c_uint(123)) + check_format(' 0000123', b'%10.7u', c_uint(123)) + check_format('0000000123', b'%010.7u', c_uint(123)) + check_format('0000123 ', b'%-10.7u', c_uint(123)) + check_format('0000123 ', b'%-010.7u', c_uint(123)) + + check_format('123', b'%2o', c_uint(0o123)) + check_format(' 123', b'%10o', c_uint(0o123)) + check_format('0000000123', b'%010o', c_uint(0o123)) + check_format('123 ', b'%-10o', c_uint(0o123)) + check_format('123 ', b'%-010o', c_uint(0o123)) + check_format('123', b'%.2o', c_uint(0o123)) + check_format('0000123', b'%.7o', c_uint(0o123)) + check_format(' 123', b'%10.2o', c_uint(0o123)) + check_format(' 0000123', b'%10.7o', c_uint(0o123)) + check_format('0000000123', b'%010.7o', c_uint(0o123)) + check_format('0000123 ', b'%-10.7o', c_uint(0o123)) + check_format('0000123 ', b'%-010.7o', c_uint(0o123)) + + check_format('abc', b'%2x', c_uint(0xabc)) + check_format(' abc', b'%10x', c_uint(0xabc)) + check_format('0000000abc', b'%010x', c_uint(0xabc)) + check_format('abc ', b'%-10x', c_uint(0xabc)) + check_format('abc ', b'%-010x', c_uint(0xabc)) + check_format('abc', b'%.2x', c_uint(0xabc)) + check_format('0000abc', b'%.7x', c_uint(0xabc)) + check_format(' abc', b'%10.2x', c_uint(0xabc)) + check_format(' 0000abc', b'%10.7x', c_uint(0xabc)) + check_format('0000000abc', b'%010.7x', c_uint(0xabc)) + check_format('0000abc ', b'%-10.7x', c_uint(0xabc)) + check_format('0000abc ', b'%-010.7x', c_uint(0xabc)) + + check_format('ABC', b'%2X', c_uint(0xabc)) + check_format(' ABC', b'%10X', c_uint(0xabc)) + check_format('0000000ABC', b'%010X', c_uint(0xabc)) + check_format('ABC ', b'%-10X', c_uint(0xabc)) + check_format('ABC ', b'%-010X', c_uint(0xabc)) + check_format('ABC', b'%.2X', c_uint(0xabc)) + check_format('0000ABC', b'%.7X', c_uint(0xabc)) + check_format(' ABC', b'%10.2X', c_uint(0xabc)) + check_format(' 0000ABC', b'%10.7X', c_uint(0xabc)) + check_format('0000000ABC', b'%010.7X', c_uint(0xabc)) + check_format('0000ABC ', b'%-10.7X', c_uint(0xabc)) + check_format('0000ABC ', b'%-010.7X', c_uint(0xabc)) # test %A check_format(r"%A:'abc\xe9\uabcd\U0010ffff'", b'%%A:%A', 'abc\xe9\uabcd\U0010ffff') # test %V - check_format('repr=abc', - b'repr=%V', 'abc', b'xyz') + check_format('abc', + b'%V', 'abc', b'xyz') + check_format('xyz', + b'%V', None, b'xyz') + + # test %ls + check_format('abc', b'%ls', c_wchar_p('abc')) + check_format('\u4eba\u6c11', b'%ls', c_wchar_p('\u4eba\u6c11')) + check_format('\U0001f4bb+\U0001f40d', + b'%ls', c_wchar_p('\U0001f4bb+\U0001f40d')) + check_format(' ab', b'%5.2ls', c_wchar_p('abc')) + check_format(' \u4eba\u6c11', b'%5ls', c_wchar_p('\u4eba\u6c11')) + check_format(' \U0001f4bb+\U0001f40d', + b'%5ls', c_wchar_p('\U0001f4bb+\U0001f40d')) + check_format('\u4eba', b'%.1ls', c_wchar_p('\u4eba\u6c11')) + check_format('\U0001f4bb' if sizeof(c_wchar) > 2 else '\ud83d', + b'%.1ls', c_wchar_p('\U0001f4bb+\U0001f40d')) + check_format('\U0001f4bb+' if sizeof(c_wchar) > 2 else '\U0001f4bb', + b'%.2ls', c_wchar_p('\U0001f4bb+\U0001f40d')) + + # test %lV + check_format('abc', + b'%lV', 'abc', c_wchar_p('xyz')) + check_format('xyz', + b'%lV', None, c_wchar_p('xyz')) + check_format('\u4eba\u6c11', + b'%lV', None, c_wchar_p('\u4eba\u6c11')) + check_format('\U0001f4bb+\U0001f40d', + b'%lV', None, c_wchar_p('\U0001f4bb+\U0001f40d')) + check_format(' ab', + b'%5.2lV', None, c_wchar_p('abc')) + check_format(' \u4eba\u6c11', + b'%5lV', None, c_wchar_p('\u4eba\u6c11')) + check_format(' \U0001f4bb+\U0001f40d', + b'%5lV', None, c_wchar_p('\U0001f4bb+\U0001f40d')) + check_format('\u4eba', + b'%.1lV', None, c_wchar_p('\u4eba\u6c11')) + check_format('\U0001f4bb' if sizeof(c_wchar) > 2 else '\ud83d', + b'%.1lV', None, c_wchar_p('\U0001f4bb+\U0001f40d')) + check_format('\U0001f4bb+' if sizeof(c_wchar) > 2 else '\U0001f4bb', + b'%.2lV', None, c_wchar_p('\U0001f4bb+\U0001f40d')) + + # test variable width and precision + check_format(' abc', b'%*s', c_int(5), b'abc') + check_format('ab', b'%.*s', c_int(2), b'abc') + check_format(' ab', b'%*.*s', c_int(5), c_int(2), b'abc') + check_format(' abc', b'%*U', c_int(5), 'abc') + check_format('ab', b'%.*U', c_int(2), 'abc') + check_format(' ab', b'%*.*U', c_int(5), c_int(2), 'abc') + check_format(' ab', b'%*.*V', c_int(5), c_int(2), None, b'abc') + check_format(' ab', b'%*.*lV', c_int(5), c_int(2), + None, c_wchar_p('abc')) + check_format(' 123', b'%*i', c_int(8), c_int(123)) + check_format('00123', b'%.*i', c_int(5), c_int(123)) + check_format(' 00123', b'%*.*i', c_int(8), c_int(5), c_int(123)) # test %p # We cannot test the exact result, @@ -564,10 +664,11 @@ def check_format(expected, format, *args): check_format('', b'%s', b'') - # check for crashes + # test invalid format strings. these tests are just here + # to check for crashes and should not be considered as specifications for fmt in (b'%', b'%0', b'%01', b'%.', b'%.1', b'%0%s', b'%1%s', b'%.%s', b'%.1%s', b'%1abc', - b'%l', b'%ll', b'%z', b'%ls', b'%lls', b'%zs'): + b'%l', b'%ll', b'%z', b'%lls', b'%zs'): with self.subTest(fmt=fmt): self.assertRaisesRegex(SystemError, 'invalid format string', PyUnicode_FromFormat, fmt, b'abc') diff --git a/Misc/NEWS.d/next/C API/2022-10-29-10-13-20.gh-issue-98836.Cy5h_z.rst b/Misc/NEWS.d/next/C API/2022-10-29-10-13-20.gh-issue-98836.Cy5h_z.rst new file mode 100644 index 00000000000000..e3730eb3f89e6c --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-10-29-10-13-20.gh-issue-98836.Cy5h_z.rst @@ -0,0 +1,4 @@ +Add support of more formatting options (left aligning, octals, uppercase +hexadecimals, :c:expr:`intmax_t`, :c:expr:`ptrdiff_t`, :c:expr:`wchar_t` C +strings, variable width and precision) in :c:func:`PyUnicode_FromFormat` and +:c:func:`PyUnicode_FromFormatV`. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 016a5a5cbca548..5bf6b3bc19b2d1 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -1330,10 +1330,8 @@ _get_peer_alt_names (_sslmodulestate *state, X509 *certificate) { p[0], p[1], p[2], p[3] ); } else if (name->d.ip->length == 16) { - /* PyUnicode_FromFormat() does not support %X */ unsigned char *p = name->d.ip->data; - len = sprintf( - buf, + v = PyUnicode_FromFormat( "%X:%X:%X:%X:%X:%X:%X:%X", p[0] << 8 | p[1], p[2] << 8 | p[3], @@ -1344,7 +1342,6 @@ _get_peer_alt_names (_sslmodulestate *state, X509 *certificate) { p[12] << 8 | p[13], p[14] << 8 | p[15] ); - v = PyUnicode_FromStringAndSize(buf, len); } else { v = PyUnicode_FromString(""); } diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index 7dd3b9c0c03e50..73929eaffc676d 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -1,3 +1,5 @@ +#include // ptrdiff_t + #define PY_SSIZE_T_CLEAN #include "parts.h" @@ -1130,25 +1132,48 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) CHECK_FORMAT_1( "%c", "c", 'c'); CHECK_FORMAT_1( "%0c", "c", 'c'); CHECK_FORMAT_1("%00c", "c", 'c'); - CHECK_FORMAT_1( "%2c", "c", 'c'); - CHECK_FORMAT_1("%02c", "c", 'c'); - CHECK_FORMAT_1("%.0c", "c", 'c'); - CHECK_FORMAT_1("%.2c", "c", 'c'); + CHECK_FORMAT_1( "%2c", NULL, 'c'); + CHECK_FORMAT_1("%02c", NULL, 'c'); + CHECK_FORMAT_1("%.0c", NULL, 'c'); + CHECK_FORMAT_1("%.2c", NULL, 'c'); // Integers CHECK_FORMAT_1("%d", "123", (int)123); CHECK_FORMAT_1("%i", "123", (int)123); CHECK_FORMAT_1("%u", "123", (unsigned int)123); + CHECK_FORMAT_1("%x", "7b", (unsigned int)123); + CHECK_FORMAT_1("%X", "7B", (unsigned int)123); + CHECK_FORMAT_1("%o", "173", (unsigned int)123); CHECK_FORMAT_1("%ld", "123", (long)123); CHECK_FORMAT_1("%li", "123", (long)123); CHECK_FORMAT_1("%lu", "123", (unsigned long)123); + CHECK_FORMAT_1("%lx", "7b", (unsigned long)123); + CHECK_FORMAT_1("%lX", "7B", (unsigned long)123); + CHECK_FORMAT_1("%lo", "173", (unsigned long)123); CHECK_FORMAT_1("%lld", "123", (long long)123); CHECK_FORMAT_1("%lli", "123", (long long)123); CHECK_FORMAT_1("%llu", "123", (unsigned long long)123); + CHECK_FORMAT_1("%llx", "7b", (unsigned long long)123); + CHECK_FORMAT_1("%llX", "7B", (unsigned long long)123); + CHECK_FORMAT_1("%llo", "173", (unsigned long long)123); CHECK_FORMAT_1("%zd", "123", (Py_ssize_t)123); CHECK_FORMAT_1("%zi", "123", (Py_ssize_t)123); CHECK_FORMAT_1("%zu", "123", (size_t)123); - CHECK_FORMAT_1("%x", "7b", (int)123); + CHECK_FORMAT_1("%zx", "7b", (size_t)123); + CHECK_FORMAT_1("%zX", "7B", (size_t)123); + CHECK_FORMAT_1("%zo", "173", (size_t)123); + CHECK_FORMAT_1("%td", "123", (ptrdiff_t)123); + CHECK_FORMAT_1("%ti", "123", (ptrdiff_t)123); + CHECK_FORMAT_1("%tu", "123", (ptrdiff_t)123); + CHECK_FORMAT_1("%tx", "7b", (ptrdiff_t)123); + CHECK_FORMAT_1("%tX", "7B", (ptrdiff_t)123); + CHECK_FORMAT_1("%to", "173", (ptrdiff_t)123); + CHECK_FORMAT_1("%jd", "123", (intmax_t)123); + CHECK_FORMAT_1("%ji", "123", (intmax_t)123); + CHECK_FORMAT_1("%ju", "123", (uintmax_t)123); + CHECK_FORMAT_1("%jx", "7b", (uintmax_t)123); + CHECK_FORMAT_1("%jX", "7B", (uintmax_t)123); + CHECK_FORMAT_1("%jo", "173", (uintmax_t)123); CHECK_FORMAT_1("%d", "-123", (int)-123); CHECK_FORMAT_1("%i", "-123", (int)-123); @@ -1158,7 +1183,10 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) CHECK_FORMAT_1("%lli", "-123", (long long)-123); CHECK_FORMAT_1("%zd", "-123", (Py_ssize_t)-123); CHECK_FORMAT_1("%zi", "-123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%x", "ffffff85", (int)-123); + CHECK_FORMAT_1("%td", "-123", (ptrdiff_t)-123); + CHECK_FORMAT_1("%ti", "-123", (ptrdiff_t)-123); + CHECK_FORMAT_1("%jd", "-123", (intmax_t)-123); + CHECK_FORMAT_1("%ji", "-123", (intmax_t)-123); // Integers: width < length CHECK_FORMAT_1("%1d", "123", (int)123); @@ -1183,7 +1211,6 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) CHECK_FORMAT_1("%1lli", "-123", (long long)-123); CHECK_FORMAT_1("%1zd", "-123", (Py_ssize_t)-123); CHECK_FORMAT_1("%1zi", "-123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%1x", "ffffff85", (int)-123); // Integers: width > length CHECK_FORMAT_1("%5d", " 123", (int)123); @@ -1208,7 +1235,6 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) CHECK_FORMAT_1("%5lli", " -123", (long long)-123); CHECK_FORMAT_1("%5zd", " -123", (Py_ssize_t)-123); CHECK_FORMAT_1("%5zi", " -123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%9x", " ffffff85", (int)-123); // Integers: width > length, 0-flag CHECK_FORMAT_1("%05d", "00123", (int)123); @@ -1233,7 +1259,6 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) CHECK_FORMAT_1("%05lli", "-0123", (long long)-123); CHECK_FORMAT_1("%05zd", "-0123", (Py_ssize_t)-123); CHECK_FORMAT_1("%05zi", "-0123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%09x", "0ffffff85", (int)-123); // Integers: precision < length CHECK_FORMAT_1("%.1d", "123", (int)123); @@ -1258,7 +1283,6 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) CHECK_FORMAT_1("%.1lli", "-123", (long long)-123); CHECK_FORMAT_1("%.1zd", "-123", (Py_ssize_t)-123); CHECK_FORMAT_1("%.1zi", "-123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%.1x", "ffffff85", (int)-123); // Integers: precision > length CHECK_FORMAT_1("%.5d", "00123", (int)123); @@ -1283,7 +1307,6 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) CHECK_FORMAT_1("%.5lli", "-00123", (long long)-123); CHECK_FORMAT_1("%.5zd", "-00123", (Py_ssize_t)-123); CHECK_FORMAT_1("%.5zi", "-00123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%.9x", "0ffffff85", (int)-123); // Integers: width > precision > length CHECK_FORMAT_1("%7.5d", " 00123", (int)123); @@ -1308,7 +1331,6 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) CHECK_FORMAT_1("%7.5lli", " -00123", (long long)-123); CHECK_FORMAT_1("%7.5zd", " -00123", (Py_ssize_t)-123); CHECK_FORMAT_1("%7.5zi", " -00123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%10.9x", " 0ffffff85", (int)-123); // Integers: width > precision > length, 0-flag CHECK_FORMAT_1("%07.5d", "0000123", (int)123); @@ -1333,7 +1355,6 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) CHECK_FORMAT_1("%07.5lli", "-000123", (long long)-123); CHECK_FORMAT_1("%07.5zd", "-000123", (Py_ssize_t)-123); CHECK_FORMAT_1("%07.5zi", "-000123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%010.9x", "00ffffff85", (int)-123); // Integers: precision > width > length CHECK_FORMAT_1("%5.7d", "0000123", (int)123); @@ -1358,7 +1379,6 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) CHECK_FORMAT_1("%5.7lli", "-0000123", (long long)-123); CHECK_FORMAT_1("%5.7zd", "-0000123", (Py_ssize_t)-123); CHECK_FORMAT_1("%5.7zi", "-0000123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%9.10x", "00ffffff85", (int)-123); // Integers: precision > width > length, 0-flag CHECK_FORMAT_1("%05.7d", "0000123", (int)123); @@ -1383,7 +1403,6 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) CHECK_FORMAT_1("%05.7lli", "-0000123", (long long)-123); CHECK_FORMAT_1("%05.7zd", "-0000123", (Py_ssize_t)-123); CHECK_FORMAT_1("%05.7zi", "-0000123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%09.10x", "00ffffff85", (int)-123); // Integers: precision = 0, arg = 0 (empty string in C) CHECK_FORMAT_1("%.0d", "0", (int)0); @@ -1402,66 +1421,80 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) // Strings CHECK_FORMAT_1("%s", "None", "None"); + CHECK_FORMAT_1("%ls", "None", L"None"); CHECK_FORMAT_1("%U", "None", unicode); CHECK_FORMAT_1("%A", "None", Py_None); CHECK_FORMAT_1("%S", "None", Py_None); CHECK_FORMAT_1("%R", "None", Py_None); CHECK_FORMAT_2("%V", "None", unicode, "ignored"); CHECK_FORMAT_2("%V", "None", NULL, "None"); + CHECK_FORMAT_2("%lV", "None", NULL, L"None"); // Strings: width < length CHECK_FORMAT_1("%1s", "None", "None"); + CHECK_FORMAT_1("%1ls", "None", L"None"); CHECK_FORMAT_1("%1U", "None", unicode); CHECK_FORMAT_1("%1A", "None", Py_None); CHECK_FORMAT_1("%1S", "None", Py_None); CHECK_FORMAT_1("%1R", "None", Py_None); CHECK_FORMAT_2("%1V", "None", unicode, "ignored"); CHECK_FORMAT_2("%1V", "None", NULL, "None"); + CHECK_FORMAT_2("%1lV", "None", NULL, L"None"); // Strings: width > length CHECK_FORMAT_1("%5s", " None", "None"); + CHECK_FORMAT_1("%5ls", " None", L"None"); CHECK_FORMAT_1("%5U", " None", unicode); CHECK_FORMAT_1("%5A", " None", Py_None); CHECK_FORMAT_1("%5S", " None", Py_None); CHECK_FORMAT_1("%5R", " None", Py_None); CHECK_FORMAT_2("%5V", " None", unicode, "ignored"); CHECK_FORMAT_2("%5V", " None", NULL, "None"); + CHECK_FORMAT_2("%5lV", " None", NULL, L"None"); // Strings: precision < length CHECK_FORMAT_1("%.1s", "N", "None"); + CHECK_FORMAT_1("%.1ls", "N", L"None"); CHECK_FORMAT_1("%.1U", "N", unicode); CHECK_FORMAT_1("%.1A", "N", Py_None); CHECK_FORMAT_1("%.1S", "N", Py_None); CHECK_FORMAT_1("%.1R", "N", Py_None); CHECK_FORMAT_2("%.1V", "N", unicode, "ignored"); CHECK_FORMAT_2("%.1V", "N", NULL, "None"); + CHECK_FORMAT_2("%.1lV", "N", NULL, L"None"); // Strings: precision > length CHECK_FORMAT_1("%.5s", "None", "None"); + CHECK_FORMAT_1("%.5ls", "None", L"None"); CHECK_FORMAT_1("%.5U", "None", unicode); CHECK_FORMAT_1("%.5A", "None", Py_None); CHECK_FORMAT_1("%.5S", "None", Py_None); CHECK_FORMAT_1("%.5R", "None", Py_None); CHECK_FORMAT_2("%.5V", "None", unicode, "ignored"); CHECK_FORMAT_2("%.5V", "None", NULL, "None"); + CHECK_FORMAT_2("%.5lV", "None", NULL, L"None"); // Strings: precision < length, width > length CHECK_FORMAT_1("%5.1s", " N", "None"); + CHECK_FORMAT_1("%5.1ls"," N", L"None"); CHECK_FORMAT_1("%5.1U", " N", unicode); CHECK_FORMAT_1("%5.1A", " N", Py_None); CHECK_FORMAT_1("%5.1S", " N", Py_None); CHECK_FORMAT_1("%5.1R", " N", Py_None); CHECK_FORMAT_2("%5.1V", " N", unicode, "ignored"); CHECK_FORMAT_2("%5.1V", " N", NULL, "None"); + CHECK_FORMAT_2("%5.1lV"," N", NULL, L"None"); // Strings: width < length, precision > length CHECK_FORMAT_1("%1.5s", "None", "None"); + CHECK_FORMAT_1("%1.5ls", "None", L"None"); CHECK_FORMAT_1("%1.5U", "None", unicode); CHECK_FORMAT_1("%1.5A", "None", Py_None); CHECK_FORMAT_1("%1.5S", "None", Py_None); CHECK_FORMAT_1("%1.5R", "None", Py_None); CHECK_FORMAT_2("%1.5V", "None", unicode, "ignored"); CHECK_FORMAT_2("%1.5V", "None", NULL, "None"); + CHECK_FORMAT_2("%1.5lV", "None", NULL, L"None"); Py_XDECREF(unicode); Py_RETURN_NONE; diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 79bd5b59ab68f9..9a4943c9eb2f75 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -1849,14 +1849,11 @@ static PyObject * kqueue_event_repr(kqueue_event_Object *s) { - char buf[1024]; - PyOS_snprintf( - buf, sizeof(buf), + return PyUnicode_FromFormat( "", (size_t)(s->e.ident), (int)s->e.filter, (unsigned int)s->e.flags, (unsigned int)s->e.fflags, (long long)(s->e.data), (void *)s->e.udata); - return PyUnicode_FromString(buf); } static int diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index c11fb4400eab2f..a86aaed501fa33 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -1339,8 +1339,6 @@ setbdaddr(const char *name, bdaddr_t *bdaddr) static PyObject * makebdaddr(bdaddr_t *bdaddr) { - char buf[(6 * 2) + 5 + 1]; - #ifdef MS_WINDOWS int i; unsigned int octets[6]; @@ -1349,16 +1347,14 @@ makebdaddr(bdaddr_t *bdaddr) octets[i] = ((*bdaddr) >> (8 * i)) & 0xFF; } - sprintf(buf, "%02X:%02X:%02X:%02X:%02X:%02X", + return PyUnicode_FromFormat("%02X:%02X:%02X:%02X:%02X:%02X", octets[5], octets[4], octets[3], octets[2], octets[1], octets[0]); #else - sprintf(buf, "%02X:%02X:%02X:%02X:%02X:%02X", + return PyUnicode_FromFormat("%02X:%02X:%02X:%02X:%02X:%02X", bdaddr->b[5], bdaddr->b[4], bdaddr->b[3], bdaddr->b[2], bdaddr->b[1], bdaddr->b[0]); #endif - - return PyUnicode_FromString(buf); } #endif diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 7726f2fb17afde..ec5684b1d09502 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -56,6 +56,7 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include "pycore_unicodeobject.h" // struct _Py_unicode_state #include "pycore_unicodeobject_generated.h" // _PyUnicode_InitStaticStrings() #include "stringlib/eq.h" // unicode_eq() +#include // ptrdiff_t #ifdef MS_WINDOWS #include @@ -2285,14 +2286,15 @@ PyUnicode_AsUCS4Copy(PyObject *string) return as_ucs4(string, NULL, 0, 1); } -/* maximum number of characters required for output of %lld or %p. - We need at most ceil(log10(256)*SIZEOF_LONG_LONG) digits, - plus 1 for the sign. 53/22 is an upper bound for log10(256). */ -#define MAX_LONG_LONG_CHARS (2 + (SIZEOF_LONG_LONG*53-1) / 22) +/* maximum number of characters required for output of %jo or %jd or %p. + We need at most ceil(log8(256)*sizeof(intmax_t)) digits, + plus 1 for the sign, plus 2 for the 0x prefix (for %p), + plus 1 for the terminal NUL. */ +#define MAX_INTMAX_CHARS (5 + (sizeof(intmax_t)*8-1) / 3) static int unicode_fromformat_write_str(_PyUnicodeWriter *writer, PyObject *str, - Py_ssize_t width, Py_ssize_t precision) + Py_ssize_t width, Py_ssize_t precision, int flags) { Py_ssize_t length, fill, arglen; Py_UCS4 maxchar; @@ -2314,8 +2316,8 @@ unicode_fromformat_write_str(_PyUnicodeWriter *writer, PyObject *str, if (_PyUnicodeWriter_Prepare(writer, arglen, maxchar) == -1) return -1; - if (width > length) { - fill = width - length; + fill = Py_MAX(width - length, 0); + if (fill && !(flags & F_LJUST)) { if (PyUnicode_Fill(writer->buffer, writer->pos, fill, ' ') == -1) return -1; writer->pos += fill; @@ -2324,12 +2326,19 @@ unicode_fromformat_write_str(_PyUnicodeWriter *writer, PyObject *str, _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, str, 0, length); writer->pos += length; + + if (fill && (flags & F_LJUST)) { + if (PyUnicode_Fill(writer->buffer, writer->pos, fill, ' ') == -1) + return -1; + writer->pos += fill; + } + return 0; } static int unicode_fromformat_write_cstr(_PyUnicodeWriter *writer, const char *str, - Py_ssize_t width, Py_ssize_t precision) + Py_ssize_t width, Py_ssize_t precision, int flags) { /* UTF-8 */ Py_ssize_t length; @@ -2349,24 +2358,58 @@ unicode_fromformat_write_cstr(_PyUnicodeWriter *writer, const char *str, if (unicode == NULL) return -1; - res = unicode_fromformat_write_str(writer, unicode, width, -1); + res = unicode_fromformat_write_str(writer, unicode, width, -1, flags); + Py_DECREF(unicode); + return res; +} + +static int +unicode_fromformat_write_wcstr(_PyUnicodeWriter *writer, const wchar_t *str, + Py_ssize_t width, Py_ssize_t precision, int flags) +{ + /* UTF-8 */ + Py_ssize_t length; + PyObject *unicode; + int res; + + if (precision == -1) { + length = wcslen(str); + } + else { + length = 0; + while (length < precision && str[length]) { + length++; + } + } + unicode = PyUnicode_FromWideChar(str, length); + if (unicode == NULL) + return -1; + + res = unicode_fromformat_write_str(writer, unicode, width, -1, flags); Py_DECREF(unicode); return res; } +#define F_LONG 1 +#define F_LONGLONG 2 +#define F_SIZE 3 +#define F_PTRDIFF 4 +#define F_INTMAX 5 +static const char * const formats[] = {"%d", "%ld", "%lld", "%zd", "%td", "%jd"}; +static const char * const formats_o[] = {"%o", "%lo", "%llo", "%zo", "%to", "%jo"}; +static const char * const formats_u[] = {"%u", "%lu", "%llu", "%zu", "%tu", "%ju"}; +static const char * const formats_x[] = {"%x", "%lx", "%llx", "%zx", "%tx", "%jx"}; +static const char * const formats_X[] = {"%X", "%lX", "%llX", "%zX", "%tX", "%jX"}; + static const char* unicode_fromformat_arg(_PyUnicodeWriter *writer, const char *f, va_list *vargs) { const char *p; Py_ssize_t len; - int zeropad; + int flags = 0; Py_ssize_t width; Py_ssize_t precision; - int longflag; - int longlongflag; - int size_tflag; - Py_ssize_t fill; p = f; f++; @@ -2377,15 +2420,31 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, return f; } - zeropad = 0; - if (*f == '0') { - zeropad = 1; - f++; + /* Parse flags. Example: "%-i" => flags=F_LJUST. */ + /* Flags '+', ' ' and '#' are not particularly useful. + * They are not worth the implementation and maintenance costs. + * In addition, '#' should add "0" for "o" conversions for compatibility + * with printf, but it would confuse Python users. */ + while (1) { + switch (*f++) { + case '-': flags |= F_LJUST; continue; + case '0': flags |= F_ZERO; continue; + } + f--; + break; } /* parse the width.precision part, e.g. "%2.5s" => width=2, precision=5 */ width = -1; - if (Py_ISDIGIT((unsigned)*f)) { + if (*f == '*') { + width = va_arg(*vargs, int); + if (width < 0) { + flags |= F_LJUST; + width = -width; + } + f++; + } + else if (Py_ISDIGIT((unsigned)*f)) { width = *f - '0'; f++; while (Py_ISDIGIT((unsigned)*f)) { @@ -2401,7 +2460,14 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, precision = -1; if (*f == '.') { f++; - if (Py_ISDIGIT((unsigned)*f)) { + if (*f == '*') { + precision = va_arg(*vargs, int); + if (precision < 0) { + precision = -2; + } + f++; + } + else if (Py_ISDIGIT((unsigned)*f)) { precision = (*f - '0'); f++; while (Py_ISDIGIT((unsigned)*f)) { @@ -2416,30 +2482,47 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, } } - /* Handle %ld, %lu, %lld and %llu. */ - longflag = 0; - longlongflag = 0; - size_tflag = 0; + int sizemod = 0; if (*f == 'l') { - if (f[1] == 'd' || f[1] == 'u' || f[1] == 'i') { - longflag = 1; - ++f; - } - else if (f[1] == 'l' && - (f[2] == 'd' || f[2] == 'u' || f[2] == 'i')) { - longlongflag = 1; + if (f[1] == 'l') { + sizemod = F_LONGLONG; f += 2; } + else { + sizemod = F_LONG; + ++f; + } } - /* handle the size_t flag. */ - else if (*f == 'z' && (f[1] == 'd' || f[1] == 'u' || f[1] == 'i')) { - size_tflag = 1; + else if (*f == 'z') { + sizemod = F_SIZE; + ++f; + } + else if (*f == 't') { + sizemod = F_PTRDIFF; + ++f; + } + else if (*f == 'j') { + sizemod = F_INTMAX; ++f; } - if (f[0] != '\0' && f[1] == '\0') writer->overallocate = 0; + switch (*f) { + case 'd': case 'i': case 'o': case 'u': case 'x': case 'X': + break; + case 'c': case 'p': + if (sizemod || width >= 0 || precision >= 0) goto invalid_format; + break; + case 's': + case 'V': + if (sizemod && sizemod != F_LONG) goto invalid_format; + break; + default: + if (sizemod) goto invalid_format; + break; + } + switch (*f) { case 'c': { @@ -2454,91 +2537,98 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, break; } - case 'i': - case 'd': - case 'u': - case 'x': + case 'd': case 'i': + case 'o': case 'u': case 'x': case 'X': { /* used by sprintf */ - char buffer[MAX_LONG_LONG_CHARS]; - Py_ssize_t arglen; - - if (*f == 'u') { - if (longflag) { - len = sprintf(buffer, "%lu", va_arg(*vargs, unsigned long)); - } - else if (longlongflag) { - len = sprintf(buffer, "%llu", va_arg(*vargs, unsigned long long)); - } - else if (size_tflag) { - len = sprintf(buffer, "%zu", va_arg(*vargs, size_t)); - } - else { - len = sprintf(buffer, "%u", va_arg(*vargs, unsigned int)); - } - } - else if (*f == 'x') { - len = sprintf(buffer, "%x", va_arg(*vargs, int)); - } - else { - if (longflag) { - len = sprintf(buffer, "%li", va_arg(*vargs, long)); - } - else if (longlongflag) { - len = sprintf(buffer, "%lli", va_arg(*vargs, long long)); - } - else if (size_tflag) { - len = sprintf(buffer, "%zi", va_arg(*vargs, Py_ssize_t)); - } - else { - len = sprintf(buffer, "%i", va_arg(*vargs, int)); - } + char buffer[MAX_INTMAX_CHARS]; + const char *fmt = NULL; + switch (*f) { + case 'o': fmt = formats_o[sizemod]; break; + case 'u': fmt = formats_u[sizemod]; break; + case 'x': fmt = formats_x[sizemod]; break; + case 'X': fmt = formats_X[sizemod]; break; + default: fmt = formats[sizemod]; break; + } + int issigned = (*f == 'd' || *f == 'i'); + switch (sizemod) { + case F_LONG: + len = issigned ? + sprintf(buffer, fmt, va_arg(*vargs, long)) : + sprintf(buffer, fmt, va_arg(*vargs, unsigned long)); + break; + case F_LONGLONG: + len = issigned ? + sprintf(buffer, fmt, va_arg(*vargs, long long)) : + sprintf(buffer, fmt, va_arg(*vargs, unsigned long long)); + break; + case F_SIZE: + len = issigned ? + sprintf(buffer, fmt, va_arg(*vargs, Py_ssize_t)) : + sprintf(buffer, fmt, va_arg(*vargs, size_t)); + break; + case F_PTRDIFF: + len = sprintf(buffer, fmt, va_arg(*vargs, ptrdiff_t)); + break; + case F_INTMAX: + len = issigned ? + sprintf(buffer, fmt, va_arg(*vargs, intmax_t)) : + sprintf(buffer, fmt, va_arg(*vargs, uintmax_t)); + break; + default: + len = issigned ? + sprintf(buffer, fmt, va_arg(*vargs, int)) : + sprintf(buffer, fmt, va_arg(*vargs, unsigned int)); + break; } assert(len >= 0); - int negative = (buffer[0] == '-'); - len -= negative; + int sign = (buffer[0] == '-'); + len -= sign; precision = Py_MAX(precision, len); - width = Py_MAX(width, precision + negative); + width = Py_MAX(width, precision + sign); + if ((flags & F_ZERO) && !(flags & F_LJUST)) { + precision = width - sign; + } - arglen = Py_MAX(precision, width); - if (_PyUnicodeWriter_Prepare(writer, arglen, 127) == -1) - return NULL; + Py_ssize_t spacepad = Py_MAX(width - precision - sign, 0); + Py_ssize_t zeropad = Py_MAX(precision - len, 0); - if (width > precision) { - if (negative && zeropad) { - if (_PyUnicodeWriter_WriteChar(writer, '-') == -1) - return NULL; - } + if (_PyUnicodeWriter_Prepare(writer, width, 127) == -1) + return NULL; - Py_UCS4 fillchar = zeropad?'0':' '; - fill = width - precision - negative; - if (PyUnicode_Fill(writer->buffer, writer->pos, fill, fillchar) == -1) + if (spacepad && !(flags & F_LJUST)) { + if (PyUnicode_Fill(writer->buffer, writer->pos, spacepad, ' ') == -1) return NULL; - writer->pos += fill; + writer->pos += spacepad; + } - if (negative && !zeropad) { - if (_PyUnicodeWriter_WriteChar(writer, '-') == -1) - return NULL; - } + if (sign) { + if (_PyUnicodeWriter_WriteChar(writer, '-') == -1) + return NULL; } - if (precision > len) { - fill = precision - len; - if (PyUnicode_Fill(writer->buffer, writer->pos, fill, '0') == -1) + if (zeropad) { + if (PyUnicode_Fill(writer->buffer, writer->pos, zeropad, '0') == -1) return NULL; - writer->pos += fill; + writer->pos += zeropad; } - if (_PyUnicodeWriter_WriteASCIIString(writer, &buffer[negative], len) < 0) + if (_PyUnicodeWriter_WriteASCIIString(writer, &buffer[sign], len) < 0) return NULL; + + if (spacepad && (flags & F_LJUST)) { + if (PyUnicode_Fill(writer->buffer, writer->pos, spacepad, ' ') == -1) + return NULL; + writer->pos += spacepad; + } break; } case 'p': { - char number[MAX_LONG_LONG_CHARS]; + char number[MAX_INTMAX_CHARS]; len = sprintf(number, "%p", va_arg(*vargs, void*)); assert(len >= 0); @@ -2561,10 +2651,17 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, case 's': { - /* UTF-8 */ - const char *s = va_arg(*vargs, const char*); - if (unicode_fromformat_write_cstr(writer, s, width, precision) < 0) - return NULL; + if (sizemod) { + const wchar_t *s = va_arg(*vargs, const wchar_t*); + if (unicode_fromformat_write_wcstr(writer, s, width, precision, flags) < 0) + return NULL; + } + else { + /* UTF-8 */ + const char *s = va_arg(*vargs, const char*); + if (unicode_fromformat_write_cstr(writer, s, width, precision, flags) < 0) + return NULL; + } break; } @@ -2573,7 +2670,7 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, PyObject *obj = va_arg(*vargs, PyObject *); assert(obj && _PyUnicode_CHECK(obj)); - if (unicode_fromformat_write_str(writer, obj, width, precision) == -1) + if (unicode_fromformat_write_str(writer, obj, width, precision, flags) == -1) return NULL; break; } @@ -2581,15 +2678,27 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, case 'V': { PyObject *obj = va_arg(*vargs, PyObject *); - const char *str = va_arg(*vargs, const char *); + const char *str; + const wchar_t *wstr; + if (sizemod) { + wstr = va_arg(*vargs, const wchar_t*); + } + else { + str = va_arg(*vargs, const char *); + } if (obj) { assert(_PyUnicode_CHECK(obj)); - if (unicode_fromformat_write_str(writer, obj, width, precision) == -1) + if (unicode_fromformat_write_str(writer, obj, width, precision, flags) == -1) + return NULL; + } + else if (sizemod) { + assert(wstr != NULL); + if (unicode_fromformat_write_wcstr(writer, wstr, width, precision, flags) < 0) return NULL; } else { assert(str != NULL); - if (unicode_fromformat_write_cstr(writer, str, width, precision) < 0) + if (unicode_fromformat_write_cstr(writer, str, width, precision, flags) < 0) return NULL; } break; @@ -2603,7 +2712,7 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, str = PyObject_Str(obj); if (!str) return NULL; - if (unicode_fromformat_write_str(writer, str, width, precision) == -1) { + if (unicode_fromformat_write_str(writer, str, width, precision, flags) == -1) { Py_DECREF(str); return NULL; } @@ -2619,7 +2728,7 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, repr = PyObject_Repr(obj); if (!repr) return NULL; - if (unicode_fromformat_write_str(writer, repr, width, precision) == -1) { + if (unicode_fromformat_write_str(writer, repr, width, precision, flags) == -1) { Py_DECREF(repr); return NULL; } @@ -2635,7 +2744,7 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, ascii = PyObject_ASCII(obj); if (!ascii) return NULL; - if (unicode_fromformat_write_str(writer, ascii, width, precision) == -1) { + if (unicode_fromformat_write_str(writer, ascii, width, precision, flags) == -1) { Py_DECREF(ascii); return NULL; } @@ -2644,6 +2753,7 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, } default: + invalid_format: PyErr_Format(PyExc_SystemError, "invalid format string: %s", p); return NULL; } diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index fb94fbeac42bae..fc4afccbfc4008 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1556,14 +1556,11 @@ verify_identifier(struct tok_state *tok) tok->cur = (char *)tok->start + PyBytes_GET_SIZE(s); } Py_DECREF(s); - // PyUnicode_FromFormatV() does not support %X - char hex[9]; - (void)PyOS_snprintf(hex, sizeof(hex), "%04X", ch); if (Py_UNICODE_ISPRINTABLE(ch)) { - syntaxerror(tok, "invalid character '%c' (U+%s)", ch, hex); + syntaxerror(tok, "invalid character '%c' (U+%04X)", ch, ch); } else { - syntaxerror(tok, "invalid non-printable character U+%s", hex); + syntaxerror(tok, "invalid non-printable character U+%04X", ch); } return 0; } @@ -2541,9 +2538,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t } if (!Py_UNICODE_ISPRINTABLE(c)) { - char hex[9]; - (void)PyOS_snprintf(hex, sizeof(hex), "%04X", c); - return MAKE_TOKEN(syntaxerror(tok, "invalid non-printable character U+%s", hex)); + return MAKE_TOKEN(syntaxerror(tok, "invalid non-printable character U+%04X", c)); } if( c == '=' && INSIDE_FSTRING_EXPR(current_tok)) { From 64d1b44a5459605af3e8572d4febfde021315633 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Mon, 22 May 2023 00:00:47 +0100 Subject: [PATCH 08/47] gh-104683: `clinic.py`: Improve coverage for the `parse_converter` method (#104729) --- Lib/test/test_clinic.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index f72cb0442f35ff..bea7303805567f 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -774,6 +774,45 @@ def test_legacy_converters(self): module, function = block.signatures self.assertIsInstance((function.parameters['path']).converter, clinic.str_converter) + def test_legacy_converters_non_string_constant_annotation(self): + expected_failure_message = """\ +Error on line 0: +Annotations must be either a name, a function call, or a string. +""" + + s = self.parse_function_should_fail('module os\nos.access\n path: 42') + self.assertEqual(s, expected_failure_message) + + s = self.parse_function_should_fail('module os\nos.access\n path: 42.42') + self.assertEqual(s, expected_failure_message) + + s = self.parse_function_should_fail('module os\nos.access\n path: 42j') + self.assertEqual(s, expected_failure_message) + + s = self.parse_function_should_fail('module os\nos.access\n path: b"42"') + self.assertEqual(s, expected_failure_message) + + def test_other_bizarre_things_in_annotations_fail(self): + expected_failure_message = """\ +Error on line 0: +Annotations must be either a name, a function call, or a string. +""" + + s = self.parse_function_should_fail( + 'module os\nos.access\n path: {"some": "dictionary"}' + ) + self.assertEqual(s, expected_failure_message) + + s = self.parse_function_should_fail( + 'module os\nos.access\n path: ["list", "of", "strings"]' + ) + self.assertEqual(s, expected_failure_message) + + s = self.parse_function_should_fail( + 'module os\nos.access\n path: (x for x in range(42))' + ) + self.assertEqual(s, expected_failure_message) + def test_unused_param(self): block = self.parse(""" module foo From cd9748409aa877d6d9905730bf68f25cf7a6a723 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 21 May 2023 16:20:19 -0700 Subject: [PATCH 09/47] gh-104686: Fix tracing for decorated classes (#104708) --- Lib/test/test_sys_settrace.py | 46 +++++++++++++++++++++++++++++++++++ Python/compile.c | 4 +++ 2 files changed, 50 insertions(+) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 0571919066bb73..5603c3cdbf3c5e 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -1524,6 +1524,52 @@ def __init__(self): (3, 'return'), (1, 'return')]) + def test_class_creation_with_decorator(self): + def func(): + def decorator(arg): + def _dec(c): + return c + return _dec + + @decorator(6) + @decorator( + len([8]), + ) + class MyObject: + pass + + self.run_and_compare(func, [ + (0, 'call'), + (1, 'line'), + (6, 'line'), + (1, 'call'), + (2, 'line'), + (4, 'line'), + (4, 'return'), + (7, 'line'), + (8, 'line'), + (7, 'line'), + (1, 'call'), + (2, 'line'), + (4, 'line'), + (4, 'return'), + (10, 'line'), + (6, 'call'), + (6, 'line'), + (11, 'line'), + (11, 'return'), + (7, 'line'), + (2, 'call'), + (3, 'line'), + (3, 'return'), + (6, 'line'), + (2, 'call'), + (3, 'line'), + (3, 'return'), + (10, 'line'), + (10, 'return'), + ]) + @support.cpython_only def test_no_line_event_after_creating_generator(self): # Spurious line events before call events only show up with C tracer diff --git a/Python/compile.c b/Python/compile.c index cfe8224a7e27c6..e4dc9729b694b0 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2486,6 +2486,10 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) } /* 2. load the 'build_class' function */ + + // these instructions should be attributed to the class line, + // not a decorator line + loc = LOC(s); ADDOP(c, loc, PUSH_NULL); ADDOP(c, loc, LOAD_BUILD_CLASS); From 4b107d86f38f6778562d4fe5e1d881b52c9d9d6c Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Mon, 22 May 2023 00:29:43 +0100 Subject: [PATCH 10/47] gh-104683: clinic.py: Modernise `parse_converter()` using pattern-matching (#104696) --- Tools/clinic/clinic.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 1d7b778da70990..d182e5e7764e46 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -5016,22 +5016,26 @@ def bad_node(self, node): key = f"{parameter_name}_as_{c_name}" if c_name else parameter_name self.function.parameters[key] = p - def parse_converter(self, annotation): - if (isinstance(annotation, ast.Constant) and - type(annotation.value) is str): - return annotation.value, True, {} + KwargDict = dict[str | None, Any] - if isinstance(annotation, ast.Name): - return annotation.id, False, {} - - if not isinstance(annotation, ast.Call): - fail("Annotations must be either a name, a function call, or a string.") - - name = annotation.func.id - symbols = globals() - - kwargs = {node.arg: eval_ast_expr(node.value, symbols) for node in annotation.keywords} - return name, False, kwargs + @staticmethod + def parse_converter(annotation: ast.expr | None) -> tuple[str, bool, KwargDict]: + match annotation: + case ast.Constant(value=str() as value): + return value, True, {} + case ast.Name(name): + return name, False, {} + case ast.Call(func=ast.Name(name)): + symbols = globals() + kwargs = { + node.arg: eval_ast_expr(node.value, symbols) + for node in annotation.keywords + } + return name, False, kwargs + case _: + fail( + "Annotations must be either a name, a function call, or a string." + ) def parse_special_symbol(self, symbol): if symbol == '*': From 8817886ae571f5b5ce4e2e6cfd2458622d0efac1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marta=20G=C3=B3mez=20Mac=C3=ADas?= Date: Mon, 22 May 2023 02:29:04 +0200 Subject: [PATCH 11/47] gh-102856: Tokenize performance improvement (#104731) --- Lib/tokenize.py | 13 +------------ Python/Python-tokenize.c | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/Lib/tokenize.py b/Lib/tokenize.py index cef2773feac24b..911f0f12f9bb7e 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -449,16 +449,6 @@ def _tokenize(rl_gen, encoding): source = b"".join(rl_gen).decode(encoding) token = None for token in _generate_tokens_from_c_tokenizer(source, extra_tokens=True): - # TODO: Marta -> limpiar esto - if 6 < token.type <= 54: - token = token._replace(type=OP) - if token.type in {ASYNC, AWAIT}: - token = token._replace(type=NAME) - if token.type == NEWLINE: - l_start, c_start = token.start - l_end, c_end = token.end - token = token._replace(string='\n', start=(l_start, c_start), end=(l_end, c_end+1)) - yield token if token is not None: last_line, _ = token.start @@ -550,8 +540,7 @@ def _generate_tokens_from_c_tokenizer(source, extra_tokens=False): """Tokenize a source reading Python code as unicode strings using the internal C tokenizer""" import _tokenize as c_tokenizer for info in c_tokenizer.TokenizerIter(source, extra_tokens=extra_tokens): - tok, type, lineno, end_lineno, col_off, end_col_off, line = info - yield TokenInfo(type, tok, (lineno, col_off), (end_lineno, end_col_off), line) + yield TokenInfo._make(info) if __name__ == "__main__": diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index ece238672e34fd..43b44be94583ee 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -207,7 +207,22 @@ tokenizeriter_next(tokenizeriterobject *it) end_col_offset = _PyPegen_byte_offset_to_character_offset(line, token.end - it->tok->line_start); } - result = Py_BuildValue("(NinnnnN)", str, type, lineno, end_lineno, col_offset, end_col_offset, line); + if (it->tok->tok_extra_tokens) { + // Necessary adjustments to match the original Python tokenize + // implementation + if (type > DEDENT && type < OP) { + type = OP; + } + else if (type == ASYNC || type == AWAIT) { + type = NAME; + } + else if (type == NEWLINE) { + str = PyUnicode_FromString("\n"); + end_col_offset++; + } + } + + result = Py_BuildValue("(iN(nn)(nn)N)", type, str, lineno, col_offset, end_lineno, end_col_offset, line); exit: _PyToken_Free(&token); return result; From b9c807a260f63284f16e25b5e98e18191f61a05f Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Mon, 22 May 2023 06:14:48 +0530 Subject: [PATCH 12/47] GH-103092: isolate `_ssl` (#104725) --- Modules/_ssl.c | 19 ++++++++++++++++--- Modules/_ssl.h | 2 ++ Modules/_ssl/debughelpers.c | 14 +++----------- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 5bf6b3bc19b2d1..59fd401cac270b 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -6150,6 +6150,18 @@ sslmodule_init_strings(PyObject *module) return 0; } +static int +sslmodule_init_lock(PyObject *module) +{ + _sslmodulestate *state = get_ssl_state(module); + state->keylog_lock = PyThread_allocate_lock(); + if (state->keylog_lock == NULL) { + PyErr_NoMemory(); + return -1; + } + return 0; +} + static PyModuleDef_Slot sslmodule_slots[] = { {Py_mod_exec, sslmodule_init_types}, {Py_mod_exec, sslmodule_init_exceptions}, @@ -6158,9 +6170,8 @@ static PyModuleDef_Slot sslmodule_slots[] = { {Py_mod_exec, sslmodule_init_constants}, {Py_mod_exec, sslmodule_init_versioninfo}, {Py_mod_exec, sslmodule_init_strings}, - // XXX gh-103092: fix isolation. - {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, - //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_exec, sslmodule_init_lock}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; @@ -6219,6 +6230,8 @@ static void sslmodule_free(void *m) { sslmodule_clear((PyObject *)m); + _sslmodulestate *state = get_ssl_state(m); + PyThread_free_lock(state->keylog_lock); } static struct PyModuleDef _sslmodule_def = { diff --git a/Modules/_ssl.h b/Modules/_ssl.h index c1da8b46b536d3..22d93ddcc6d6eb 100644 --- a/Modules/_ssl.h +++ b/Modules/_ssl.h @@ -33,6 +33,8 @@ typedef struct { PyObject *str_reason; PyObject *str_verify_code; PyObject *str_verify_message; + /* keylog lock */ + PyThread_type_lock keylog_lock; } _sslmodulestate; static struct PyModuleDef _sslmodule_def; diff --git a/Modules/_ssl/debughelpers.c b/Modules/_ssl/debughelpers.c index 217f224942556e..a81f0aad05a802 100644 --- a/Modules/_ssl/debughelpers.c +++ b/Modules/_ssl/debughelpers.c @@ -118,30 +118,22 @@ _PySSL_keylog_callback(const SSL *ssl, const char *line) PyGILState_STATE threadstate; PySSLSocket *ssl_obj = NULL; /* ssl._SSLSocket, borrowed ref */ int res, e; - static PyThread_type_lock *lock = NULL; threadstate = PyGILState_Ensure(); ssl_obj = (PySSLSocket *)SSL_get_app_data(ssl); assert(Py_IS_TYPE(ssl_obj, get_state_sock(ssl_obj)->PySSLSocket_Type)); + PyThread_type_lock lock = get_state_sock(ssl_obj)->keylog_lock; + assert(lock != NULL); if (ssl_obj->ctx->keylog_bio == NULL) { return; } - - /* Allocate a static lock to synchronize writes to keylog file. + /* * The lock is neither released on exit nor on fork(). The lock is * also shared between all SSLContexts although contexts may write to * their own files. IMHO that's good enough for a non-performance * critical debug helper. */ - if (lock == NULL) { - lock = PyThread_allocate_lock(); - if (lock == NULL) { - PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); - ssl_obj->exc = PyErr_GetRaisedException(); - return; - } - } PySSL_BEGIN_ALLOW_THREADS PyThread_acquire_lock(lock, 1); From ef5d00a59207a63c6d5ae0d5d44054847d1bf3b5 Mon Sep 17 00:00:00 2001 From: Luccccifer Date: Mon, 22 May 2023 11:48:57 +0800 Subject: [PATCH 13/47] gh-104536: Improve `multiprocessing.process._cleanup` logic (#104537) Fix a race condition in the internal `multiprocessing.process` cleanup logic that could manifest as an unintended `AttributeError` when calling `BaseProcess.close()`. --------- Co-authored-by: Oleg Iarygin Co-authored-by: Gregory P. Smith --- Lib/multiprocessing/process.py | 2 +- .../Library/2023-05-16-10-07-16.gh-issue-104536.hFWD8f.rst | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-05-16-10-07-16.gh-issue-104536.hFWD8f.rst diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index c03c859baa795b..271ba3fd325138 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -61,7 +61,7 @@ def parent_process(): def _cleanup(): # check for processes which have finished for p in list(_children): - if p._popen.poll() is not None: + if (child_popen := p._popen) and child_popen.poll() is not None: _children.discard(p) # diff --git a/Misc/NEWS.d/next/Library/2023-05-16-10-07-16.gh-issue-104536.hFWD8f.rst b/Misc/NEWS.d/next/Library/2023-05-16-10-07-16.gh-issue-104536.hFWD8f.rst new file mode 100644 index 00000000000000..b0f5d78f7e61da --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-16-10-07-16.gh-issue-104536.hFWD8f.rst @@ -0,0 +1,3 @@ +Fix a race condition in the internal :mod:`multiprocessing.process` cleanup +logic that could manifest as an unintended ``AttributeError`` when calling +``process.close()``. From a5f244d627a6815cf2d8ccec836b9b52eb3e8de2 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 21 May 2023 21:25:09 -0700 Subject: [PATCH 14/47] gh-104656: Rename typeparams AST node to type_params (#104657) --- Doc/library/ast.rst | 6 +- Grammar/python.gram | 8 +- Include/internal/pycore_ast.h | 53 ++-- Include/internal/pycore_ast_state.h | 4 +- Lib/ast.py | 12 +- ...-04-25-08-43-11.gh-issue-103763.ZLBZk1.rst | 2 +- Parser/Python.asdl | 16 +- Parser/action_helpers.c | 6 +- Parser/parser.c | 30 +-- Python/Python-ast.c | 252 +++++++++--------- Python/ast.c | 18 +- Python/ast_opt.c | 12 +- Python/compile.c | 72 ++--- Python/symtable.c | 86 +++--- 14 files changed, 290 insertions(+), 287 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index eb6a973cac62d2..b6b1e076c9f08c 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -1724,7 +1724,7 @@ Function and class definitions body=[ FunctionDef( name='f', - typeparams=[], + type_params=[], args=arguments( posonlyargs=[], args=[ @@ -1848,7 +1848,7 @@ Function and class definitions body=[ ClassDef( name='Foo', - typeparams=[], + type_params=[], bases=[ Name(id='base1', ctx=Load()), Name(id='base2', ctx=Load())], @@ -1887,7 +1887,7 @@ Async and await body=[ AsyncFunctionDef( name='f', - typeparams=[], + type_params=[], args=arguments( posonlyargs=[], args=[], diff --git a/Grammar/python.gram b/Grammar/python.gram index c79207b9cb51a4..e6a983429e39d8 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -640,12 +640,12 @@ type_alias[stmt_ty]: # Type parameter declaration # -------------------------- -type_params[asdl_typeparam_seq*]: '[' t=type_param_seq ']' { - CHECK_VERSION(asdl_typeparam_seq *, 12, "Type parameter lists are", t) } +type_params[asdl_type_param_seq*]: '[' t=type_param_seq ']' { + CHECK_VERSION(asdl_type_param_seq *, 12, "Type parameter lists are", t) } -type_param_seq[asdl_typeparam_seq*]: a[asdl_typeparam_seq*]=','.type_param+ [','] { a } +type_param_seq[asdl_type_param_seq*]: a[asdl_type_param_seq*]=','.type_param+ [','] { a } -type_param[typeparam_ty] (memo): +type_param[type_param_ty] (memo): | a=NAME b=[type_param_bound] { _PyAST_TypeVar(a->v.Name.id, b, EXTRA) } | '*' a=NAME colon=":" e=expression { RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind diff --git a/Include/internal/pycore_ast.h b/Include/internal/pycore_ast.h index 9f1cef0541508c..06a40239a2473a 100644 --- a/Include/internal/pycore_ast.h +++ b/Include/internal/pycore_ast.h @@ -51,7 +51,7 @@ typedef struct _pattern *pattern_ty; typedef struct _type_ignore *type_ignore_ty; -typedef struct _typeparam *typeparam_ty; +typedef struct _type_param *type_param_ty; typedef struct { @@ -151,10 +151,11 @@ asdl_type_ignore_seq *_Py_asdl_type_ignore_seq_new(Py_ssize_t size, PyArena typedef struct { _ASDL_SEQ_HEAD - typeparam_ty typed_elements[1]; -} asdl_typeparam_seq; + type_param_ty typed_elements[1]; +} asdl_type_param_seq; -asdl_typeparam_seq *_Py_asdl_typeparam_seq_new(Py_ssize_t size, PyArena *arena); +asdl_type_param_seq *_Py_asdl_type_param_seq_new(Py_ssize_t size, PyArena + *arena); enum _mod_kind {Module_kind=1, Interactive_kind=2, Expression_kind=3, @@ -197,7 +198,7 @@ struct _stmt { union { struct { identifier name; - asdl_typeparam_seq *typeparams; + asdl_type_param_seq *type_params; arguments_ty args; asdl_stmt_seq *body; asdl_expr_seq *decorator_list; @@ -207,7 +208,7 @@ struct _stmt { struct { identifier name; - asdl_typeparam_seq *typeparams; + asdl_type_param_seq *type_params; arguments_ty args; asdl_stmt_seq *body; asdl_expr_seq *decorator_list; @@ -217,7 +218,7 @@ struct _stmt { struct { identifier name; - asdl_typeparam_seq *typeparams; + asdl_type_param_seq *type_params; asdl_expr_seq *bases; asdl_keyword_seq *keywords; asdl_stmt_seq *body; @@ -240,7 +241,7 @@ struct _stmt { struct { expr_ty name; - asdl_typeparam_seq *typeparams; + asdl_type_param_seq *type_params; expr_ty value; } TypeAlias; @@ -649,9 +650,9 @@ struct _type_ignore { } v; }; -enum _typeparam_kind {TypeVar_kind=1, ParamSpec_kind=2, TypeVarTuple_kind=3}; -struct _typeparam { - enum _typeparam_kind kind; +enum _type_param_kind {TypeVar_kind=1, ParamSpec_kind=2, TypeVarTuple_kind=3}; +struct _type_param { + enum _type_param_kind kind; union { struct { identifier name; @@ -681,18 +682,18 @@ mod_ty _PyAST_Interactive(asdl_stmt_seq * body, PyArena *arena); mod_ty _PyAST_Expression(expr_ty body, PyArena *arena); mod_ty _PyAST_FunctionType(asdl_expr_seq * argtypes, expr_ty returns, PyArena *arena); -stmt_ty _PyAST_FunctionDef(identifier name, asdl_typeparam_seq * typeparams, +stmt_ty _PyAST_FunctionDef(identifier name, asdl_type_param_seq * type_params, arguments_ty args, asdl_stmt_seq * body, asdl_expr_seq * decorator_list, expr_ty returns, string type_comment, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); -stmt_ty _PyAST_AsyncFunctionDef(identifier name, asdl_typeparam_seq * - typeparams, arguments_ty args, asdl_stmt_seq * +stmt_ty _PyAST_AsyncFunctionDef(identifier name, asdl_type_param_seq * + type_params, arguments_ty args, asdl_stmt_seq * body, asdl_expr_seq * decorator_list, expr_ty returns, string type_comment, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); -stmt_ty _PyAST_ClassDef(identifier name, asdl_typeparam_seq * typeparams, +stmt_ty _PyAST_ClassDef(identifier name, asdl_type_param_seq * type_params, asdl_expr_seq * bases, asdl_keyword_seq * keywords, asdl_stmt_seq * body, asdl_expr_seq * decorator_list, int lineno, int col_offset, int end_lineno, int @@ -704,9 +705,9 @@ stmt_ty _PyAST_Delete(asdl_expr_seq * targets, int lineno, int col_offset, int stmt_ty _PyAST_Assign(asdl_expr_seq * targets, expr_ty value, string type_comment, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); -stmt_ty _PyAST_TypeAlias(expr_ty name, asdl_typeparam_seq * typeparams, expr_ty - value, int lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena); +stmt_ty _PyAST_TypeAlias(expr_ty name, asdl_type_param_seq * type_params, + expr_ty value, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); stmt_ty _PyAST_AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); @@ -891,14 +892,14 @@ pattern_ty _PyAST_MatchOr(asdl_pattern_seq * patterns, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); type_ignore_ty _PyAST_TypeIgnore(int lineno, string tag, PyArena *arena); -typeparam_ty _PyAST_TypeVar(identifier name, expr_ty bound, int lineno, int - col_offset, int end_lineno, int end_col_offset, - PyArena *arena); -typeparam_ty _PyAST_ParamSpec(identifier name, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena); -typeparam_ty _PyAST_TypeVarTuple(identifier name, int lineno, int col_offset, - int end_lineno, int end_col_offset, PyArena - *arena); +type_param_ty _PyAST_TypeVar(identifier name, expr_ty bound, int lineno, int + col_offset, int end_lineno, int end_col_offset, + PyArena *arena); +type_param_ty _PyAST_ParamSpec(identifier name, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +type_param_ty _PyAST_TypeVarTuple(identifier name, int lineno, int col_offset, + int end_lineno, int end_col_offset, PyArena + *arena); PyObject* PyAST_mod2obj(mod_ty t); diff --git a/Include/internal/pycore_ast_state.h b/Include/internal/pycore_ast_state.h index e723ead577b888..0c0d53f3e5d7e9 100644 --- a/Include/internal/pycore_ast_state.h +++ b/Include/internal/pycore_ast_state.h @@ -248,8 +248,8 @@ struct ast_state { PyObject *type_comment; PyObject *type_ignore_type; PyObject *type_ignores; - PyObject *typeparam_type; - PyObject *typeparams; + PyObject *type_param_type; + PyObject *type_params; PyObject *unaryop_type; PyObject *upper; PyObject *value; diff --git a/Lib/ast.py b/Lib/ast.py index 08904afb203183..226910ecac0b4a 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -1051,7 +1051,7 @@ def visit_ClassDef(self, node): self.fill("@") self.traverse(deco) self.fill("class " + node.name) - self._typeparams_helper(node.typeparams) + self._type_params_helper(node.type_params) with self.delimit_if("(", ")", condition = node.bases or node.keywords): comma = False for e in node.bases: @@ -1083,7 +1083,7 @@ def _function_helper(self, node, fill_suffix): self.traverse(deco) def_str = fill_suffix + " " + node.name self.fill(def_str) - self._typeparams_helper(node.typeparams) + self._type_params_helper(node.type_params) with self.delimit("(", ")"): self.traverse(node.args) if node.returns: @@ -1092,10 +1092,10 @@ def _function_helper(self, node, fill_suffix): with self.block(extra=self.get_type_comment(node)): self._write_docstring_and_traverse_body(node) - def _typeparams_helper(self, typeparams): - if typeparams is not None and len(typeparams) > 0: + def _type_params_helper(self, type_params): + if type_params is not None and len(type_params) > 0: with self.delimit("[", "]"): - self.interleave(lambda: self.write(", "), self.traverse, typeparams) + self.interleave(lambda: self.write(", "), self.traverse, type_params) def visit_TypeVar(self, node): self.write(node.name) @@ -1112,7 +1112,7 @@ def visit_ParamSpec(self, node): def visit_TypeAlias(self, node): self.fill("type ") self.traverse(node.name) - self._typeparams_helper(node.typeparams) + self._type_params_helper(node.type_params) self.write(" = ") self.traverse(node.value) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-25-08-43-11.gh-issue-103763.ZLBZk1.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-25-08-43-11.gh-issue-103763.ZLBZk1.rst index cefe6429ab2041..cb8d2565012658 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-25-08-43-11.gh-issue-103763.ZLBZk1.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-04-25-08-43-11.gh-issue-103763.ZLBZk1.rst @@ -8,7 +8,7 @@ attribute. This is implemented as a new AST node ``ast.TypeAlias``. New syntax (``class X[T]: ...``, ``def func[T](): ...``) is added for defining generic functions and classes. This is implemented as a new -``typeparams`` attribute on the AST nodes for classes and functions. +``type_params`` attribute on the AST nodes for classes and functions. This node holds instances of the new AST classes ``ast.TypeVar``, ``ast.ParamSpec``, and ``ast.TypeVarTuple``. diff --git a/Parser/Python.asdl b/Parser/Python.asdl index cfc41ef45b568b..dc2eb802b0436c 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -8,15 +8,15 @@ module Python | Expression(expr body) | FunctionType(expr* argtypes, expr returns) - stmt = FunctionDef(identifier name, typeparam* typeparams, arguments args, + stmt = FunctionDef(identifier name, type_param* type_params, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment) - | AsyncFunctionDef(identifier name, typeparam* typeparams, arguments args, + | AsyncFunctionDef(identifier name, type_param* type_params, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment) | ClassDef(identifier name, - typeparam* typeparams, + type_param* type_params, expr* bases, keyword* keywords, stmt* body, @@ -25,7 +25,7 @@ module Python | Delete(expr* targets) | Assign(expr* targets, expr value, string? type_comment) - | TypeAlias(expr name, typeparam* typeparams, expr value) + | TypeAlias(expr name, type_param* type_params, expr value) | AugAssign(expr target, operator op, expr value) -- 'simple' indicates that we annotate simple name without parens | AnnAssign(expr target, expr annotation, expr? value, int simple) @@ -145,8 +145,8 @@ module Python type_ignore = TypeIgnore(int lineno, string tag) - typeparam = TypeVar(identifier name, expr? bound) - | ParamSpec(identifier name) - | TypeVarTuple(identifier name) - attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) + type_param = TypeVar(identifier name, expr? bound) + | ParamSpec(identifier name) + | TypeVarTuple(identifier name) + attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) } diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 0134e6f16ba8f0..06d77b64cacbcc 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -752,7 +752,7 @@ _PyPegen_function_def_decorators(Parser *p, asdl_expr_seq *decorators, stmt_ty f assert(function_def != NULL); if (function_def->kind == AsyncFunctionDef_kind) { return _PyAST_AsyncFunctionDef( - function_def->v.FunctionDef.name, function_def->v.FunctionDef.typeparams, + function_def->v.FunctionDef.name, function_def->v.FunctionDef.type_params, function_def->v.FunctionDef.args, function_def->v.FunctionDef.body, decorators, function_def->v.FunctionDef.returns, function_def->v.FunctionDef.type_comment, function_def->lineno, @@ -761,7 +761,7 @@ _PyPegen_function_def_decorators(Parser *p, asdl_expr_seq *decorators, stmt_ty f } return _PyAST_FunctionDef( - function_def->v.FunctionDef.name, function_def->v.FunctionDef.typeparams, + function_def->v.FunctionDef.name, function_def->v.FunctionDef.type_params, function_def->v.FunctionDef.args, function_def->v.FunctionDef.body, decorators, function_def->v.FunctionDef.returns, @@ -776,7 +776,7 @@ _PyPegen_class_def_decorators(Parser *p, asdl_expr_seq *decorators, stmt_ty clas { assert(class_def != NULL); return _PyAST_ClassDef( - class_def->v.ClassDef.name, class_def->v.ClassDef.typeparams, + class_def->v.ClassDef.name, class_def->v.ClassDef.type_params, class_def->v.ClassDef.bases, class_def->v.ClassDef.keywords, class_def->v.ClassDef.body, decorators, class_def->lineno, class_def->col_offset, class_def->end_lineno, diff --git a/Parser/parser.c b/Parser/parser.c index 894846714eff94..fc5466fea2b3fc 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -696,9 +696,9 @@ static asdl_pattern_seq* positional_patterns_rule(Parser *p); static asdl_seq* keyword_patterns_rule(Parser *p); static KeyPatternPair* keyword_pattern_rule(Parser *p); static stmt_ty type_alias_rule(Parser *p); -static asdl_typeparam_seq* type_params_rule(Parser *p); -static asdl_typeparam_seq* type_param_seq_rule(Parser *p); -static typeparam_ty type_param_rule(Parser *p); +static asdl_type_param_seq* type_params_rule(Parser *p); +static asdl_type_param_seq* type_param_seq_rule(Parser *p); +static type_param_ty type_param_rule(Parser *p); static expr_ty type_param_bound_rule(Parser *p); static expr_ty expressions_rule(Parser *p); static expr_ty expression_rule(Parser *p); @@ -10653,7 +10653,7 @@ type_alias_rule(Parser *p) } // type_params: '[' type_param_seq ']' -static asdl_typeparam_seq* +static asdl_type_param_seq* type_params_rule(Parser *p) { if (p->level++ == MAXSTACK) { @@ -10664,7 +10664,7 @@ type_params_rule(Parser *p) p->level--; return NULL; } - asdl_typeparam_seq* _res = NULL; + asdl_type_param_seq* _res = NULL; int _mark = p->mark; { // '[' type_param_seq ']' if (p->error_indicator) { @@ -10674,7 +10674,7 @@ type_params_rule(Parser *p) D(fprintf(stderr, "%*c> type_params[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'[' type_param_seq ']'")); Token * _literal; Token * _literal_1; - asdl_typeparam_seq* t; + asdl_type_param_seq* t; if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' && @@ -10684,7 +10684,7 @@ type_params_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ type_params[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'[' type_param_seq ']'")); - _res = CHECK_VERSION ( asdl_typeparam_seq* , 12 , "Type parameter lists are" , t ); + _res = CHECK_VERSION ( asdl_type_param_seq* , 12 , "Type parameter lists are" , t ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -10703,7 +10703,7 @@ type_params_rule(Parser *p) } // type_param_seq: ','.type_param+ ','? -static asdl_typeparam_seq* +static asdl_type_param_seq* type_param_seq_rule(Parser *p) { if (p->level++ == MAXSTACK) { @@ -10714,7 +10714,7 @@ type_param_seq_rule(Parser *p) p->level--; return NULL; } - asdl_typeparam_seq* _res = NULL; + asdl_type_param_seq* _res = NULL; int _mark = p->mark; { // ','.type_param+ ','? if (p->error_indicator) { @@ -10724,9 +10724,9 @@ type_param_seq_rule(Parser *p) D(fprintf(stderr, "%*c> type_param_seq[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.type_param+ ','?")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - asdl_typeparam_seq* a; + asdl_type_param_seq* a; if ( - (a = (asdl_typeparam_seq*)_gather_81_rule(p)) // ','.type_param+ + (a = (asdl_type_param_seq*)_gather_81_rule(p)) // ','.type_param+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -10756,7 +10756,7 @@ type_param_seq_rule(Parser *p) // | '*' NAME // | '**' NAME ":" expression // | '**' NAME -static typeparam_ty +static type_param_ty type_param_rule(Parser *p) { if (p->level++ == MAXSTACK) { @@ -10767,7 +10767,7 @@ type_param_rule(Parser *p) p->level--; return NULL; } - typeparam_ty _res = NULL; + type_param_ty _res = NULL; if (_PyPegen_is_memoized(p, type_param_type, &_res)) { p->level--; return _res; @@ -30199,7 +30199,7 @@ _loop0_82_rule(Parser *p) } D(fprintf(stderr, "%*c> _loop0_82[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' type_param")); Token * _literal; - typeparam_ty elem; + type_param_ty elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -30266,7 +30266,7 @@ _gather_81_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _gather_81[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "type_param _loop0_82")); - typeparam_ty elem; + type_param_ty elem; asdl_seq * seq; if ( (elem = type_param_rule(p)) // type_param diff --git a/Python/Python-ast.c b/Python/Python-ast.c index d62cccbb6e2415..87906d975d7414 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -261,8 +261,8 @@ void _PyAST_Fini(PyInterpreterState *interp) Py_CLEAR(state->type_comment); Py_CLEAR(state->type_ignore_type); Py_CLEAR(state->type_ignores); - Py_CLEAR(state->typeparam_type); - Py_CLEAR(state->typeparams); + Py_CLEAR(state->type_param_type); + Py_CLEAR(state->type_params); Py_CLEAR(state->unaryop_type); Py_CLEAR(state->upper); Py_CLEAR(state->value); @@ -362,7 +362,7 @@ static int init_identifiers(struct ast_state *state) if ((state->type = PyUnicode_InternFromString("type")) == NULL) return 0; if ((state->type_comment = PyUnicode_InternFromString("type_comment")) == NULL) return 0; if ((state->type_ignores = PyUnicode_InternFromString("type_ignores")) == NULL) return 0; - if ((state->typeparams = PyUnicode_InternFromString("typeparams")) == NULL) return 0; + if ((state->type_params = PyUnicode_InternFromString("type_params")) == NULL) return 0; if ((state->upper = PyUnicode_InternFromString("upper")) == NULL) return 0; if ((state->value = PyUnicode_InternFromString("value")) == NULL) return 0; if ((state->values = PyUnicode_InternFromString("values")) == NULL) return 0; @@ -383,7 +383,7 @@ GENERATE_ASDL_SEQ_CONSTRUCTOR(withitem, withitem_ty) GENERATE_ASDL_SEQ_CONSTRUCTOR(match_case, match_case_ty) GENERATE_ASDL_SEQ_CONSTRUCTOR(pattern, pattern_ty) GENERATE_ASDL_SEQ_CONSTRUCTOR(type_ignore, type_ignore_ty) -GENERATE_ASDL_SEQ_CONSTRUCTOR(typeparam, typeparam_ty) +GENERATE_ASDL_SEQ_CONSTRUCTOR(type_param, type_param_ty) static PyObject* ast2obj_mod(struct ast_state *state, void*); static const char * const Module_fields[]={ @@ -409,7 +409,7 @@ static const char * const stmt_attributes[] = { static PyObject* ast2obj_stmt(struct ast_state *state, void*); static const char * const FunctionDef_fields[]={ "name", - "typeparams", + "type_params", "args", "body", "decorator_list", @@ -418,7 +418,7 @@ static const char * const FunctionDef_fields[]={ }; static const char * const AsyncFunctionDef_fields[]={ "name", - "typeparams", + "type_params", "args", "body", "decorator_list", @@ -427,7 +427,7 @@ static const char * const AsyncFunctionDef_fields[]={ }; static const char * const ClassDef_fields[]={ "name", - "typeparams", + "type_params", "bases", "keywords", "body", @@ -446,7 +446,7 @@ static const char * const Assign_fields[]={ }; static const char * const TypeAlias_fields[]={ "name", - "typeparams", + "type_params", "value", }; static const char * const AugAssign_fields[]={ @@ -775,13 +775,13 @@ static const char * const TypeIgnore_fields[]={ "lineno", "tag", }; -static const char * const typeparam_attributes[] = { +static const char * const type_param_attributes[] = { "lineno", "col_offset", "end_lineno", "end_col_offset", }; -static PyObject* ast2obj_typeparam(struct ast_state *state, void*); +static PyObject* ast2obj_type_param(struct ast_state *state, void*); static const char * const TypeVar_fields[]={ "name", "bound", @@ -1169,13 +1169,13 @@ init_types(struct ast_state *state) "FunctionType(expr* argtypes, expr returns)"); if (!state->FunctionType_type) return 0; state->stmt_type = make_type(state, "stmt", state->AST_type, NULL, 0, - "stmt = FunctionDef(identifier name, typeparam* typeparams, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment)\n" - " | AsyncFunctionDef(identifier name, typeparam* typeparams, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment)\n" - " | ClassDef(identifier name, typeparam* typeparams, expr* bases, keyword* keywords, stmt* body, expr* decorator_list)\n" + "stmt = FunctionDef(identifier name, type_param* type_params, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment)\n" + " | AsyncFunctionDef(identifier name, type_param* type_params, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment)\n" + " | ClassDef(identifier name, type_param* type_params, expr* bases, keyword* keywords, stmt* body, expr* decorator_list)\n" " | Return(expr? value)\n" " | Delete(expr* targets)\n" " | Assign(expr* targets, expr value, string? type_comment)\n" - " | TypeAlias(expr name, typeparam* typeparams, expr value)\n" + " | TypeAlias(expr name, type_param* type_params, expr value)\n" " | AugAssign(expr target, operator op, expr value)\n" " | AnnAssign(expr target, expr annotation, expr? value, int simple)\n" " | For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)\n" @@ -1206,7 +1206,7 @@ init_types(struct ast_state *state) return 0; state->FunctionDef_type = make_type(state, "FunctionDef", state->stmt_type, FunctionDef_fields, 7, - "FunctionDef(identifier name, typeparam* typeparams, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment)"); + "FunctionDef(identifier name, type_param* type_params, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment)"); if (!state->FunctionDef_type) return 0; if (PyObject_SetAttr(state->FunctionDef_type, state->returns, Py_None) == -1) @@ -1217,7 +1217,7 @@ init_types(struct ast_state *state) state->AsyncFunctionDef_type = make_type(state, "AsyncFunctionDef", state->stmt_type, AsyncFunctionDef_fields, 7, - "AsyncFunctionDef(identifier name, typeparam* typeparams, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment)"); + "AsyncFunctionDef(identifier name, type_param* type_params, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment)"); if (!state->AsyncFunctionDef_type) return 0; if (PyObject_SetAttr(state->AsyncFunctionDef_type, state->returns, Py_None) == -1) @@ -1227,7 +1227,7 @@ init_types(struct ast_state *state) return 0; state->ClassDef_type = make_type(state, "ClassDef", state->stmt_type, ClassDef_fields, 6, - "ClassDef(identifier name, typeparam* typeparams, expr* bases, keyword* keywords, stmt* body, expr* decorator_list)"); + "ClassDef(identifier name, type_param* type_params, expr* bases, keyword* keywords, stmt* body, expr* decorator_list)"); if (!state->ClassDef_type) return 0; state->Return_type = make_type(state, "Return", state->stmt_type, Return_fields, 1, @@ -1248,7 +1248,7 @@ init_types(struct ast_state *state) return 0; state->TypeAlias_type = make_type(state, "TypeAlias", state->stmt_type, TypeAlias_fields, 3, - "TypeAlias(expr name, typeparam* typeparams, expr value)"); + "TypeAlias(expr name, type_param* type_params, expr value)"); if (!state->TypeAlias_type) return 0; state->AugAssign_type = make_type(state, "AugAssign", state->stmt_type, AugAssign_fields, 3, @@ -1894,33 +1894,33 @@ init_types(struct ast_state *state) TypeIgnore_fields, 2, "TypeIgnore(int lineno, string tag)"); if (!state->TypeIgnore_type) return 0; - state->typeparam_type = make_type(state, "typeparam", state->AST_type, - NULL, 0, - "typeparam = TypeVar(identifier name, expr? bound)\n" - " | ParamSpec(identifier name)\n" - " | TypeVarTuple(identifier name)"); - if (!state->typeparam_type) return 0; - if (!add_attributes(state, state->typeparam_type, typeparam_attributes, 4)) - return 0; - if (PyObject_SetAttr(state->typeparam_type, state->end_lineno, Py_None) == + state->type_param_type = make_type(state, "type_param", state->AST_type, + NULL, 0, + "type_param = TypeVar(identifier name, expr? bound)\n" + " | ParamSpec(identifier name)\n" + " | TypeVarTuple(identifier name)"); + if (!state->type_param_type) return 0; + if (!add_attributes(state, state->type_param_type, type_param_attributes, + 4)) return 0; + if (PyObject_SetAttr(state->type_param_type, state->end_lineno, Py_None) == -1) return 0; - if (PyObject_SetAttr(state->typeparam_type, state->end_col_offset, Py_None) - == -1) + if (PyObject_SetAttr(state->type_param_type, state->end_col_offset, + Py_None) == -1) return 0; - state->TypeVar_type = make_type(state, "TypeVar", state->typeparam_type, + state->TypeVar_type = make_type(state, "TypeVar", state->type_param_type, TypeVar_fields, 2, "TypeVar(identifier name, expr? bound)"); if (!state->TypeVar_type) return 0; if (PyObject_SetAttr(state->TypeVar_type, state->bound, Py_None) == -1) return 0; state->ParamSpec_type = make_type(state, "ParamSpec", - state->typeparam_type, ParamSpec_fields, + state->type_param_type, ParamSpec_fields, 1, "ParamSpec(identifier name)"); if (!state->ParamSpec_type) return 0; state->TypeVarTuple_type = make_type(state, "TypeVarTuple", - state->typeparam_type, + state->type_param_type, TypeVarTuple_fields, 1, "TypeVarTuple(identifier name)"); if (!state->TypeVarTuple_type) return 0; @@ -1967,8 +1967,8 @@ static int obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, PyArena* arena); static int obj2ast_type_ignore(struct ast_state *state, PyObject* obj, type_ignore_ty* out, PyArena* arena); -static int obj2ast_typeparam(struct ast_state *state, PyObject* obj, - typeparam_ty* out, PyArena* arena); +static int obj2ast_type_param(struct ast_state *state, PyObject* obj, + type_param_ty* out, PyArena* arena); mod_ty _PyAST_Module(asdl_stmt_seq * body, asdl_type_ignore_seq * type_ignores, @@ -2032,7 +2032,7 @@ _PyAST_FunctionType(asdl_expr_seq * argtypes, expr_ty returns, PyArena *arena) } stmt_ty -_PyAST_FunctionDef(identifier name, asdl_typeparam_seq * typeparams, +_PyAST_FunctionDef(identifier name, asdl_type_param_seq * type_params, arguments_ty args, asdl_stmt_seq * body, asdl_expr_seq * decorator_list, expr_ty returns, string type_comment, int lineno, int col_offset, int end_lineno, int end_col_offset, @@ -2054,7 +2054,7 @@ _PyAST_FunctionDef(identifier name, asdl_typeparam_seq * typeparams, return NULL; p->kind = FunctionDef_kind; p->v.FunctionDef.name = name; - p->v.FunctionDef.typeparams = typeparams; + p->v.FunctionDef.type_params = type_params; p->v.FunctionDef.args = args; p->v.FunctionDef.body = body; p->v.FunctionDef.decorator_list = decorator_list; @@ -2068,7 +2068,7 @@ _PyAST_FunctionDef(identifier name, asdl_typeparam_seq * typeparams, } stmt_ty -_PyAST_AsyncFunctionDef(identifier name, asdl_typeparam_seq * typeparams, +_PyAST_AsyncFunctionDef(identifier name, asdl_type_param_seq * type_params, arguments_ty args, asdl_stmt_seq * body, asdl_expr_seq * decorator_list, expr_ty returns, string type_comment, int lineno, int col_offset, int end_lineno, int @@ -2090,7 +2090,7 @@ _PyAST_AsyncFunctionDef(identifier name, asdl_typeparam_seq * typeparams, return NULL; p->kind = AsyncFunctionDef_kind; p->v.AsyncFunctionDef.name = name; - p->v.AsyncFunctionDef.typeparams = typeparams; + p->v.AsyncFunctionDef.type_params = type_params; p->v.AsyncFunctionDef.args = args; p->v.AsyncFunctionDef.body = body; p->v.AsyncFunctionDef.decorator_list = decorator_list; @@ -2104,10 +2104,11 @@ _PyAST_AsyncFunctionDef(identifier name, asdl_typeparam_seq * typeparams, } stmt_ty -_PyAST_ClassDef(identifier name, asdl_typeparam_seq * typeparams, asdl_expr_seq - * bases, asdl_keyword_seq * keywords, asdl_stmt_seq * body, - asdl_expr_seq * decorator_list, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena) +_PyAST_ClassDef(identifier name, asdl_type_param_seq * type_params, + asdl_expr_seq * bases, asdl_keyword_seq * keywords, + asdl_stmt_seq * body, asdl_expr_seq * decorator_list, int + lineno, int col_offset, int end_lineno, int end_col_offset, + PyArena *arena) { stmt_ty p; if (!name) { @@ -2120,7 +2121,7 @@ _PyAST_ClassDef(identifier name, asdl_typeparam_seq * typeparams, asdl_expr_seq return NULL; p->kind = ClassDef_kind; p->v.ClassDef.name = name; - p->v.ClassDef.typeparams = typeparams; + p->v.ClassDef.type_params = type_params; p->v.ClassDef.bases = bases; p->v.ClassDef.keywords = keywords; p->v.ClassDef.body = body; @@ -2192,8 +2193,8 @@ _PyAST_Assign(asdl_expr_seq * targets, expr_ty value, string type_comment, int } stmt_ty -_PyAST_TypeAlias(expr_ty name, asdl_typeparam_seq * typeparams, expr_ty value, - int lineno, int col_offset, int end_lineno, int +_PyAST_TypeAlias(expr_ty name, asdl_type_param_seq * type_params, expr_ty + value, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena) { stmt_ty p; @@ -2212,7 +2213,7 @@ _PyAST_TypeAlias(expr_ty name, asdl_typeparam_seq * typeparams, expr_ty value, return NULL; p->kind = TypeAlias_kind; p->v.TypeAlias.name = name; - p->v.TypeAlias.typeparams = typeparams; + p->v.TypeAlias.type_params = type_params; p->v.TypeAlias.value = value; p->lineno = lineno; p->col_offset = col_offset; @@ -3713,17 +3714,17 @@ _PyAST_TypeIgnore(int lineno, string tag, PyArena *arena) return p; } -typeparam_ty +type_param_ty _PyAST_TypeVar(identifier name, expr_ty bound, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena) { - typeparam_ty p; + type_param_ty p; if (!name) { PyErr_SetString(PyExc_ValueError, "field 'name' is required for TypeVar"); return NULL; } - p = (typeparam_ty)_PyArena_Malloc(arena, sizeof(*p)); + p = (type_param_ty)_PyArena_Malloc(arena, sizeof(*p)); if (!p) return NULL; p->kind = TypeVar_kind; @@ -3736,17 +3737,17 @@ _PyAST_TypeVar(identifier name, expr_ty bound, int lineno, int col_offset, int return p; } -typeparam_ty +type_param_ty _PyAST_ParamSpec(identifier name, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena) { - typeparam_ty p; + type_param_ty p; if (!name) { PyErr_SetString(PyExc_ValueError, "field 'name' is required for ParamSpec"); return NULL; } - p = (typeparam_ty)_PyArena_Malloc(arena, sizeof(*p)); + p = (type_param_ty)_PyArena_Malloc(arena, sizeof(*p)); if (!p) return NULL; p->kind = ParamSpec_kind; @@ -3758,17 +3759,17 @@ _PyAST_ParamSpec(identifier name, int lineno, int col_offset, int end_lineno, return p; } -typeparam_ty +type_param_ty _PyAST_TypeVarTuple(identifier name, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena) { - typeparam_ty p; + type_param_ty p; if (!name) { PyErr_SetString(PyExc_ValueError, "field 'name' is required for TypeVarTuple"); return NULL; } - p = (typeparam_ty)_PyArena_Malloc(arena, sizeof(*p)); + p = (type_param_ty)_PyArena_Malloc(arena, sizeof(*p)); if (!p) return NULL; p->kind = TypeVarTuple_kind; @@ -3882,10 +3883,10 @@ ast2obj_stmt(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.FunctionDef.typeparams, - ast2obj_typeparam); + value = ast2obj_list(state, (asdl_seq*)o->v.FunctionDef.type_params, + ast2obj_type_param); if (!value) goto failed; - if (PyObject_SetAttr(result, state->typeparams, value) == -1) + if (PyObject_SetAttr(result, state->type_params, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_arguments(state, o->v.FunctionDef.args); @@ -3926,10 +3927,10 @@ ast2obj_stmt(struct ast_state *state, void* _o) goto failed; Py_DECREF(value); value = ast2obj_list(state, - (asdl_seq*)o->v.AsyncFunctionDef.typeparams, - ast2obj_typeparam); + (asdl_seq*)o->v.AsyncFunctionDef.type_params, + ast2obj_type_param); if (!value) goto failed; - if (PyObject_SetAttr(result, state->typeparams, value) == -1) + if (PyObject_SetAttr(result, state->type_params, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_arguments(state, o->v.AsyncFunctionDef.args); @@ -3970,10 +3971,10 @@ ast2obj_stmt(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.ClassDef.typeparams, - ast2obj_typeparam); + value = ast2obj_list(state, (asdl_seq*)o->v.ClassDef.type_params, + ast2obj_type_param); if (!value) goto failed; - if (PyObject_SetAttr(result, state->typeparams, value) == -1) + if (PyObject_SetAttr(result, state->type_params, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(state, (asdl_seq*)o->v.ClassDef.bases, @@ -4052,10 +4053,10 @@ ast2obj_stmt(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.TypeAlias.typeparams, - ast2obj_typeparam); + value = ast2obj_list(state, (asdl_seq*)o->v.TypeAlias.type_params, + ast2obj_type_param); if (!value) goto failed; - if (PyObject_SetAttr(result, state->typeparams, value) == -1) + if (PyObject_SetAttr(result, state->type_params, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr(state, o->v.TypeAlias.value); @@ -5656,9 +5657,9 @@ ast2obj_type_ignore(struct ast_state *state, void* _o) } PyObject* -ast2obj_typeparam(struct ast_state *state, void* _o) +ast2obj_type_param(struct ast_state *state, void* _o) { - typeparam_ty o = (typeparam_ty)_o; + type_param_ty o = (type_param_ty)_o; PyObject *result = NULL, *value = NULL; PyTypeObject *tp; if (!o) { @@ -6074,7 +6075,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* } if (isinstance) { identifier name; - asdl_typeparam_seq* typeparams; + asdl_type_param_seq* type_params; arguments_ty args; asdl_stmt_seq* body; asdl_expr_seq* decorator_list; @@ -6098,11 +6099,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (res != 0) goto failed; Py_CLEAR(tmp); } - if (_PyObject_LookupAttr(obj, state->typeparams, &tmp) < 0) { + if (_PyObject_LookupAttr(obj, state->type_params, &tmp) < 0) { return 1; } if (tmp == NULL) { - PyErr_SetString(PyExc_TypeError, "required field \"typeparams\" missing from FunctionDef"); + PyErr_SetString(PyExc_TypeError, "required field \"type_params\" missing from FunctionDef"); return 1; } else { @@ -6110,27 +6111,27 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_ssize_t len; Py_ssize_t i; if (!PyList_Check(tmp)) { - PyErr_Format(PyExc_TypeError, "FunctionDef field \"typeparams\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); + PyErr_Format(PyExc_TypeError, "FunctionDef field \"type_params\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); goto failed; } len = PyList_GET_SIZE(tmp); - typeparams = _Py_asdl_typeparam_seq_new(len, arena); - if (typeparams == NULL) goto failed; + type_params = _Py_asdl_type_param_seq_new(len, arena); + if (type_params == NULL) goto failed; for (i = 0; i < len; i++) { - typeparam_ty val; + type_param_ty val; PyObject *tmp2 = Py_NewRef(PyList_GET_ITEM(tmp, i)); if (_Py_EnterRecursiveCall(" while traversing 'FunctionDef' node")) { goto failed; } - res = obj2ast_typeparam(state, tmp2, &val, arena); + res = obj2ast_type_param(state, tmp2, &val, arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { - PyErr_SetString(PyExc_RuntimeError, "FunctionDef field \"typeparams\" changed size during iteration"); + PyErr_SetString(PyExc_RuntimeError, "FunctionDef field \"type_params\" changed size during iteration"); goto failed; } - asdl_seq_SET(typeparams, i, val); + asdl_seq_SET(type_params, i, val); } Py_CLEAR(tmp); } @@ -6257,9 +6258,10 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = _PyAST_FunctionDef(name, typeparams, args, body, decorator_list, - returns, type_comment, lineno, col_offset, - end_lineno, end_col_offset, arena); + *out = _PyAST_FunctionDef(name, type_params, args, body, + decorator_list, returns, type_comment, + lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -6270,7 +6272,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* } if (isinstance) { identifier name; - asdl_typeparam_seq* typeparams; + asdl_type_param_seq* type_params; arguments_ty args; asdl_stmt_seq* body; asdl_expr_seq* decorator_list; @@ -6294,11 +6296,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (res != 0) goto failed; Py_CLEAR(tmp); } - if (_PyObject_LookupAttr(obj, state->typeparams, &tmp) < 0) { + if (_PyObject_LookupAttr(obj, state->type_params, &tmp) < 0) { return 1; } if (tmp == NULL) { - PyErr_SetString(PyExc_TypeError, "required field \"typeparams\" missing from AsyncFunctionDef"); + PyErr_SetString(PyExc_TypeError, "required field \"type_params\" missing from AsyncFunctionDef"); return 1; } else { @@ -6306,27 +6308,27 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_ssize_t len; Py_ssize_t i; if (!PyList_Check(tmp)) { - PyErr_Format(PyExc_TypeError, "AsyncFunctionDef field \"typeparams\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); + PyErr_Format(PyExc_TypeError, "AsyncFunctionDef field \"type_params\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); goto failed; } len = PyList_GET_SIZE(tmp); - typeparams = _Py_asdl_typeparam_seq_new(len, arena); - if (typeparams == NULL) goto failed; + type_params = _Py_asdl_type_param_seq_new(len, arena); + if (type_params == NULL) goto failed; for (i = 0; i < len; i++) { - typeparam_ty val; + type_param_ty val; PyObject *tmp2 = Py_NewRef(PyList_GET_ITEM(tmp, i)); if (_Py_EnterRecursiveCall(" while traversing 'AsyncFunctionDef' node")) { goto failed; } - res = obj2ast_typeparam(state, tmp2, &val, arena); + res = obj2ast_type_param(state, tmp2, &val, arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { - PyErr_SetString(PyExc_RuntimeError, "AsyncFunctionDef field \"typeparams\" changed size during iteration"); + PyErr_SetString(PyExc_RuntimeError, "AsyncFunctionDef field \"type_params\" changed size during iteration"); goto failed; } - asdl_seq_SET(typeparams, i, val); + asdl_seq_SET(type_params, i, val); } Py_CLEAR(tmp); } @@ -6453,7 +6455,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = _PyAST_AsyncFunctionDef(name, typeparams, args, body, + *out = _PyAST_AsyncFunctionDef(name, type_params, args, body, decorator_list, returns, type_comment, lineno, col_offset, end_lineno, end_col_offset, arena); @@ -6467,7 +6469,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* } if (isinstance) { identifier name; - asdl_typeparam_seq* typeparams; + asdl_type_param_seq* type_params; asdl_expr_seq* bases; asdl_keyword_seq* keywords; asdl_stmt_seq* body; @@ -6490,11 +6492,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (res != 0) goto failed; Py_CLEAR(tmp); } - if (_PyObject_LookupAttr(obj, state->typeparams, &tmp) < 0) { + if (_PyObject_LookupAttr(obj, state->type_params, &tmp) < 0) { return 1; } if (tmp == NULL) { - PyErr_SetString(PyExc_TypeError, "required field \"typeparams\" missing from ClassDef"); + PyErr_SetString(PyExc_TypeError, "required field \"type_params\" missing from ClassDef"); return 1; } else { @@ -6502,27 +6504,27 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_ssize_t len; Py_ssize_t i; if (!PyList_Check(tmp)) { - PyErr_Format(PyExc_TypeError, "ClassDef field \"typeparams\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); + PyErr_Format(PyExc_TypeError, "ClassDef field \"type_params\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); goto failed; } len = PyList_GET_SIZE(tmp); - typeparams = _Py_asdl_typeparam_seq_new(len, arena); - if (typeparams == NULL) goto failed; + type_params = _Py_asdl_type_param_seq_new(len, arena); + if (type_params == NULL) goto failed; for (i = 0; i < len; i++) { - typeparam_ty val; + type_param_ty val; PyObject *tmp2 = Py_NewRef(PyList_GET_ITEM(tmp, i)); if (_Py_EnterRecursiveCall(" while traversing 'ClassDef' node")) { goto failed; } - res = obj2ast_typeparam(state, tmp2, &val, arena); + res = obj2ast_type_param(state, tmp2, &val, arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { - PyErr_SetString(PyExc_RuntimeError, "ClassDef field \"typeparams\" changed size during iteration"); + PyErr_SetString(PyExc_RuntimeError, "ClassDef field \"type_params\" changed size during iteration"); goto failed; } - asdl_seq_SET(typeparams, i, val); + asdl_seq_SET(type_params, i, val); } Py_CLEAR(tmp); } @@ -6670,7 +6672,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* } Py_CLEAR(tmp); } - *out = _PyAST_ClassDef(name, typeparams, bases, keywords, body, + *out = _PyAST_ClassDef(name, type_params, bases, keywords, body, decorator_list, lineno, col_offset, end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; @@ -6847,7 +6849,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* } if (isinstance) { expr_ty name; - asdl_typeparam_seq* typeparams; + asdl_type_param_seq* type_params; expr_ty value; if (_PyObject_LookupAttr(obj, state->name, &tmp) < 0) { @@ -6867,11 +6869,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (res != 0) goto failed; Py_CLEAR(tmp); } - if (_PyObject_LookupAttr(obj, state->typeparams, &tmp) < 0) { + if (_PyObject_LookupAttr(obj, state->type_params, &tmp) < 0) { return 1; } if (tmp == NULL) { - PyErr_SetString(PyExc_TypeError, "required field \"typeparams\" missing from TypeAlias"); + PyErr_SetString(PyExc_TypeError, "required field \"type_params\" missing from TypeAlias"); return 1; } else { @@ -6879,27 +6881,27 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_ssize_t len; Py_ssize_t i; if (!PyList_Check(tmp)) { - PyErr_Format(PyExc_TypeError, "TypeAlias field \"typeparams\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); + PyErr_Format(PyExc_TypeError, "TypeAlias field \"type_params\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); goto failed; } len = PyList_GET_SIZE(tmp); - typeparams = _Py_asdl_typeparam_seq_new(len, arena); - if (typeparams == NULL) goto failed; + type_params = _Py_asdl_type_param_seq_new(len, arena); + if (type_params == NULL) goto failed; for (i = 0; i < len; i++) { - typeparam_ty val; + type_param_ty val; PyObject *tmp2 = Py_NewRef(PyList_GET_ITEM(tmp, i)); if (_Py_EnterRecursiveCall(" while traversing 'TypeAlias' node")) { goto failed; } - res = obj2ast_typeparam(state, tmp2, &val, arena); + res = obj2ast_type_param(state, tmp2, &val, arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { - PyErr_SetString(PyExc_RuntimeError, "TypeAlias field \"typeparams\" changed size during iteration"); + PyErr_SetString(PyExc_RuntimeError, "TypeAlias field \"type_params\" changed size during iteration"); goto failed; } - asdl_seq_SET(typeparams, i, val); + asdl_seq_SET(type_params, i, val); } Py_CLEAR(tmp); } @@ -6920,7 +6922,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = _PyAST_TypeAlias(name, typeparams, value, lineno, col_offset, + *out = _PyAST_TypeAlias(name, type_params, value, lineno, col_offset, end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; @@ -12293,8 +12295,8 @@ obj2ast_type_ignore(struct ast_state *state, PyObject* obj, type_ignore_ty* } int -obj2ast_typeparam(struct ast_state *state, PyObject* obj, typeparam_ty* out, - PyArena* arena) +obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, + PyArena* arena) { int isinstance; @@ -12313,12 +12315,12 @@ obj2ast_typeparam(struct ast_state *state, PyObject* obj, typeparam_ty* out, return 1; } if (tmp == NULL) { - PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from typeparam"); + PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from type_param"); return 1; } else { int res; - if (_Py_EnterRecursiveCall(" while traversing 'typeparam' node")) { + if (_Py_EnterRecursiveCall(" while traversing 'type_param' node")) { goto failed; } res = obj2ast_int(state, tmp, &lineno, arena); @@ -12330,12 +12332,12 @@ obj2ast_typeparam(struct ast_state *state, PyObject* obj, typeparam_ty* out, return 1; } if (tmp == NULL) { - PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from typeparam"); + PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from type_param"); return 1; } else { int res; - if (_Py_EnterRecursiveCall(" while traversing 'typeparam' node")) { + if (_Py_EnterRecursiveCall(" while traversing 'type_param' node")) { goto failed; } res = obj2ast_int(state, tmp, &col_offset, arena); @@ -12352,7 +12354,7 @@ obj2ast_typeparam(struct ast_state *state, PyObject* obj, typeparam_ty* out, } else { int res; - if (_Py_EnterRecursiveCall(" while traversing 'typeparam' node")) { + if (_Py_EnterRecursiveCall(" while traversing 'type_param' node")) { goto failed; } res = obj2ast_int(state, tmp, &end_lineno, arena); @@ -12369,7 +12371,7 @@ obj2ast_typeparam(struct ast_state *state, PyObject* obj, typeparam_ty* out, } else { int res; - if (_Py_EnterRecursiveCall(" while traversing 'typeparam' node")) { + if (_Py_EnterRecursiveCall(" while traversing 'type_param' node")) { goto failed; } res = obj2ast_int(state, tmp, &end_col_offset, arena); @@ -12486,7 +12488,7 @@ obj2ast_typeparam(struct ast_state *state, PyObject* obj, typeparam_ty* out, return 0; } - PyErr_Format(PyExc_TypeError, "expected some sort of typeparam, but got %R", obj); + PyErr_Format(PyExc_TypeError, "expected some sort of type_param, but got %R", obj); failed: Py_XDECREF(tmp); return 1; @@ -12880,7 +12882,7 @@ astmodule_exec(PyObject *m) if (PyModule_AddObjectRef(m, "TypeIgnore", state->TypeIgnore_type) < 0) { return -1; } - if (PyModule_AddObjectRef(m, "typeparam", state->typeparam_type) < 0) { + if (PyModule_AddObjectRef(m, "type_param", state->type_param_type) < 0) { return -1; } if (PyModule_AddObjectRef(m, "TypeVar", state->TypeVar_type) < 0) { diff --git a/Python/ast.c b/Python/ast.c index 0844f2afa06bfe..68600ce683b974 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -17,12 +17,12 @@ struct validator { static int validate_stmts(struct validator *, asdl_stmt_seq *); static int validate_exprs(struct validator *, asdl_expr_seq *, expr_context_ty, int); static int validate_patterns(struct validator *, asdl_pattern_seq *, int); -static int validate_typeparams(struct validator *, asdl_typeparam_seq *); +static int validate_type_params(struct validator *, asdl_type_param_seq *); static int _validate_nonempty_seq(asdl_seq *, const char *, const char *); static int validate_stmt(struct validator *, stmt_ty); static int validate_expr(struct validator *, expr_ty, expr_context_ty); static int validate_pattern(struct validator *, pattern_ty, int); -static int validate_typeparam(struct validator *, typeparam_ty); +static int validate_typeparam(struct validator *, type_param_ty); #define VALIDATE_POSITIONS(node) \ if (node->lineno > node->end_lineno) { \ @@ -728,7 +728,7 @@ validate_stmt(struct validator *state, stmt_ty stmt) switch (stmt->kind) { case FunctionDef_kind: ret = validate_body(state, stmt->v.FunctionDef.body, "FunctionDef") && - validate_typeparams(state, stmt->v.FunctionDef.typeparams) && + validate_type_params(state, stmt->v.FunctionDef.type_params) && validate_arguments(state, stmt->v.FunctionDef.args) && validate_exprs(state, stmt->v.FunctionDef.decorator_list, Load, 0) && (!stmt->v.FunctionDef.returns || @@ -736,7 +736,7 @@ validate_stmt(struct validator *state, stmt_ty stmt) break; case ClassDef_kind: ret = validate_body(state, stmt->v.ClassDef.body, "ClassDef") && - validate_typeparams(state, stmt->v.ClassDef.typeparams) && + validate_type_params(state, stmt->v.ClassDef.type_params) && validate_exprs(state, stmt->v.ClassDef.bases, Load, 0) && validate_keywords(state, stmt->v.ClassDef.keywords) && validate_exprs(state, stmt->v.ClassDef.decorator_list, Load, 0); @@ -769,7 +769,7 @@ validate_stmt(struct validator *state, stmt_ty stmt) break; case TypeAlias_kind: ret = validate_expr(state, stmt->v.TypeAlias.name, Store) && - validate_typeparams(state, stmt->v.TypeAlias.typeparams) && + validate_type_params(state, stmt->v.TypeAlias.type_params) && validate_expr(state, stmt->v.TypeAlias.value, Load); break; case For_kind: @@ -919,7 +919,7 @@ validate_stmt(struct validator *state, stmt_ty stmt) break; case AsyncFunctionDef_kind: ret = validate_body(state, stmt->v.AsyncFunctionDef.body, "AsyncFunctionDef") && - validate_typeparams(state, stmt->v.AsyncFunctionDef.typeparams) && + validate_type_params(state, stmt->v.AsyncFunctionDef.type_params) && validate_arguments(state, stmt->v.AsyncFunctionDef.args) && validate_exprs(state, stmt->v.AsyncFunctionDef.decorator_list, Load, 0) && (!stmt->v.AsyncFunctionDef.returns || @@ -993,7 +993,7 @@ validate_patterns(struct validator *state, asdl_pattern_seq *patterns, int star_ } static int -validate_typeparam(struct validator *state, typeparam_ty tp) +validate_typeparam(struct validator *state, type_param_ty tp) { VALIDATE_POSITIONS(tp); int ret = -1; @@ -1014,11 +1014,11 @@ validate_typeparam(struct validator *state, typeparam_ty tp) } static int -validate_typeparams(struct validator *state, asdl_typeparam_seq *tps) +validate_type_params(struct validator *state, asdl_type_param_seq *tps) { Py_ssize_t i; for (i = 0; i < asdl_seq_LEN(tps); i++) { - typeparam_ty tp = asdl_seq_GET(tps, i); + type_param_ty tp = asdl_seq_GET(tps, i); if (tp) { if (!validate_typeparam(state, tp)) return 0; diff --git a/Python/ast_opt.c b/Python/ast_opt.c index c5b3e075467334..274bd134e1435b 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -642,7 +642,7 @@ static int astfold_withitem(withitem_ty node_, PyArena *ctx_, _PyASTOptimizeStat static int astfold_excepthandler(excepthandler_ty node_, PyArena *ctx_, _PyASTOptimizeState *state); static int astfold_match_case(match_case_ty node_, PyArena *ctx_, _PyASTOptimizeState *state); static int astfold_pattern(pattern_ty node_, PyArena *ctx_, _PyASTOptimizeState *state); -static int astfold_typeparam(typeparam_ty node_, PyArena *ctx_, _PyASTOptimizeState *state); +static int astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimizeState *state); #define CALL(FUNC, TYPE, ARG) \ if (!FUNC((ARG), ctx_, state)) \ @@ -881,7 +881,7 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) } switch (node_->kind) { case FunctionDef_kind: - CALL_SEQ(astfold_typeparam, typeparam, node_->v.FunctionDef.typeparams); + CALL_SEQ(astfold_type_param, type_param, node_->v.FunctionDef.type_params); CALL(astfold_arguments, arguments_ty, node_->v.FunctionDef.args); CALL(astfold_body, asdl_seq, node_->v.FunctionDef.body); CALL_SEQ(astfold_expr, expr, node_->v.FunctionDef.decorator_list); @@ -890,7 +890,7 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) } break; case AsyncFunctionDef_kind: - CALL_SEQ(astfold_typeparam, typeparam, node_->v.AsyncFunctionDef.typeparams); + CALL_SEQ(astfold_type_param, type_param, node_->v.AsyncFunctionDef.type_params); CALL(astfold_arguments, arguments_ty, node_->v.AsyncFunctionDef.args); CALL(astfold_body, asdl_seq, node_->v.AsyncFunctionDef.body); CALL_SEQ(astfold_expr, expr, node_->v.AsyncFunctionDef.decorator_list); @@ -899,7 +899,7 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) } break; case ClassDef_kind: - CALL_SEQ(astfold_typeparam, typeparam, node_->v.ClassDef.typeparams); + CALL_SEQ(astfold_type_param, type_param, node_->v.ClassDef.type_params); CALL_SEQ(astfold_expr, expr, node_->v.ClassDef.bases); CALL_SEQ(astfold_keyword, keyword, node_->v.ClassDef.keywords); CALL(astfold_body, asdl_seq, node_->v.ClassDef.body); @@ -928,7 +928,7 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) break; case TypeAlias_kind: CALL(astfold_expr, expr_ty, node_->v.TypeAlias.name); - CALL_SEQ(astfold_typeparam, typeparam, node_->v.TypeAlias.typeparams); + CALL_SEQ(astfold_type_param, type_param, node_->v.TypeAlias.type_params); CALL(astfold_expr, expr_ty, node_->v.TypeAlias.value); break; case For_kind: @@ -1084,7 +1084,7 @@ astfold_match_case(match_case_ty node_, PyArena *ctx_, _PyASTOptimizeState *stat } static int -astfold_typeparam(typeparam_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) +astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) { switch (node_->kind) { case TypeVar_kind: diff --git a/Python/compile.c b/Python/compile.c index e4dc9729b694b0..f2314ae11c417b 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2113,15 +2113,15 @@ wrap_in_stopiteration_handler(struct compiler *c) } static int -compiler_type_params(struct compiler *c, asdl_typeparam_seq *typeparams) +compiler_type_params(struct compiler *c, asdl_type_param_seq *type_params) { - if (!typeparams) { + if (!type_params) { return SUCCESS; } - Py_ssize_t n = asdl_seq_LEN(typeparams); + Py_ssize_t n = asdl_seq_LEN(type_params); for (Py_ssize_t i = 0; i < n; i++) { - typeparam_ty typeparam = asdl_seq_GET(typeparams, i); + type_param_ty typeparam = asdl_seq_GET(type_params, i); location loc = LOC(typeparam); switch(typeparam->kind) { case TypeVar_kind: @@ -2170,7 +2170,7 @@ compiler_type_params(struct compiler *c, asdl_typeparam_seq *typeparams) break; } } - ADDOP_I(c, LOC(asdl_seq_GET(typeparams, 0)), BUILD_TUPLE, n); + ADDOP_I(c, LOC(asdl_seq_GET(type_params, 0)), BUILD_TUPLE, n); return SUCCESS; } @@ -2248,7 +2248,7 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) expr_ty returns; identifier name; asdl_expr_seq *decos; - asdl_typeparam_seq *typeparams; + asdl_type_param_seq *type_params; Py_ssize_t funcflags; int annotations; int firstlineno; @@ -2260,7 +2260,7 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) returns = s->v.AsyncFunctionDef.returns; decos = s->v.AsyncFunctionDef.decorator_list; name = s->v.AsyncFunctionDef.name; - typeparams = s->v.AsyncFunctionDef.typeparams; + type_params = s->v.AsyncFunctionDef.type_params; } else { assert(s->kind == FunctionDef_kind); @@ -2268,7 +2268,7 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) returns = s->v.FunctionDef.returns; decos = s->v.FunctionDef.decorator_list; name = s->v.FunctionDef.name; - typeparams = s->v.FunctionDef.typeparams; + type_params = s->v.FunctionDef.type_params; } RETURN_IF_ERROR(compiler_check_debug_args(c, args)); @@ -2281,7 +2281,7 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) location loc = LOC(s); - int is_generic = asdl_seq_LEN(typeparams) > 0; + int is_generic = asdl_seq_LEN(type_params) > 0; if (is_generic) { // Used by the CALL to the type parameters function. @@ -2305,17 +2305,17 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) if (num_typeparam_args == 2) { ADDOP_I(c, loc, SWAP, 2); } - PyObject *typeparams_name = PyUnicode_FromFormat("", name); - if (!typeparams_name) { + PyObject *type_params_name = PyUnicode_FromFormat("", name); + if (!type_params_name) { return ERROR; } - if (compiler_enter_scope(c, typeparams_name, COMPILER_SCOPE_TYPEPARAMS, - (void *)typeparams, firstlineno) == -1) { - Py_DECREF(typeparams_name); + if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS, + (void *)type_params, firstlineno) == -1) { + Py_DECREF(type_params_name); return ERROR; } - Py_DECREF(typeparams_name); - RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, typeparams)); + Py_DECREF(type_params_name); + RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, type_params)); if ((funcflags & 0x01) || (funcflags & 0x02)) { RETURN_IF_ERROR_IN_SCOPE(c, codegen_addop_i(INSTR_SEQUENCE(c), LOAD_FAST, 0, loc)); } @@ -2416,8 +2416,8 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) compiler_exit_scope(c); return ERROR; } - asdl_typeparam_seq *typeparams = s->v.ClassDef.typeparams; - if (asdl_seq_LEN(typeparams) > 0) { + asdl_type_param_seq *type_params = s->v.ClassDef.type_params; + if (asdl_seq_LEN(type_params) > 0) { if (!compiler_set_type_params_in_class(c, loc)) { compiler_exit_scope(c); return ERROR; @@ -2519,23 +2519,23 @@ compiler_class(struct compiler *c, stmt_ty s) } location loc = LOC(s); - asdl_typeparam_seq *typeparams = s->v.ClassDef.typeparams; - int is_generic = asdl_seq_LEN(typeparams) > 0; + asdl_type_param_seq *type_params = s->v.ClassDef.type_params; + int is_generic = asdl_seq_LEN(type_params) > 0; if (is_generic) { Py_XSETREF(c->u->u_private, Py_NewRef(s->v.ClassDef.name)); ADDOP(c, loc, PUSH_NULL); - PyObject *typeparams_name = PyUnicode_FromFormat("", + PyObject *type_params_name = PyUnicode_FromFormat("", s->v.ClassDef.name); - if (!typeparams_name) { + if (!type_params_name) { return ERROR; } - if (compiler_enter_scope(c, typeparams_name, COMPILER_SCOPE_TYPEPARAMS, - (void *)typeparams, firstlineno) == -1) { - Py_DECREF(typeparams_name); + if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS, + (void *)type_params, firstlineno) == -1) { + Py_DECREF(type_params_name); return ERROR; } - Py_DECREF(typeparams_name); - RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, typeparams)); + Py_DECREF(type_params_name); + RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, type_params)); _Py_DECLARE_STR(type_params, ".type_params"); RETURN_IF_ERROR_IN_SCOPE(c, compiler_nameop(c, loc, &_Py_STR(type_params), Store)); } @@ -2637,26 +2637,26 @@ static int compiler_typealias(struct compiler *c, stmt_ty s) { location loc = LOC(s); - asdl_typeparam_seq *typeparams = s->v.TypeAlias.typeparams; - int is_generic = asdl_seq_LEN(typeparams) > 0; + asdl_type_param_seq *type_params = s->v.TypeAlias.type_params; + int is_generic = asdl_seq_LEN(type_params) > 0; PyObject *name = s->v.TypeAlias.name->v.Name.id; if (is_generic) { ADDOP(c, loc, PUSH_NULL); - PyObject *typeparams_name = PyUnicode_FromFormat("", + PyObject *type_params_name = PyUnicode_FromFormat("", name); - if (!typeparams_name) { + if (!type_params_name) { return ERROR; } - if (compiler_enter_scope(c, typeparams_name, COMPILER_SCOPE_TYPEPARAMS, - (void *)typeparams, loc.lineno) == -1) { - Py_DECREF(typeparams_name); + if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS, + (void *)type_params, loc.lineno) == -1) { + Py_DECREF(type_params_name); return ERROR; } - Py_DECREF(typeparams_name); + Py_DECREF(type_params_name); RETURN_IF_ERROR_IN_SCOPE( c, compiler_addop_load_const(c->c_const_cache, c->u, loc, name) ); - RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, typeparams)); + RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, type_params)); } else { ADDOP_LOAD_CONST(c, loc, name); diff --git a/Python/symtable.c b/Python/symtable.c index 73cbb2b8e995b2..e2c00d17480dd1 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -231,7 +231,7 @@ static int symtable_enter_block(struct symtable *st, identifier name, static int symtable_exit_block(struct symtable *st); static int symtable_visit_stmt(struct symtable *st, stmt_ty s); static int symtable_visit_expr(struct symtable *st, expr_ty s); -static int symtable_visit_typeparam(struct symtable *st, typeparam_ty s); +static int symtable_visit_type_param(struct symtable *st, type_param_ty s); static int symtable_visit_genexp(struct symtable *st, expr_ty s); static int symtable_visit_listcomp(struct symtable *st, expr_ty s); static int symtable_visit_setcomp(struct symtable *st, expr_ty s); @@ -528,7 +528,7 @@ error_at_directive(PySTEntryObject *ste, PyObject *name) static int analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, PyObject *bound, PyObject *local, PyObject *free, - PyObject *global, PyObject *typeparams, PySTEntryObject *class_entry) + PyObject *global, PyObject *type_params, PySTEntryObject *class_entry) { if (flags & DEF_GLOBAL) { if (flags & DEF_NONLOCAL) { @@ -557,7 +557,7 @@ analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, return error_at_directive(ste, name); } - if (PySet_Contains(typeparams, name)) { + if (PySet_Contains(type_params, name)) { PyErr_Format(PyExc_SyntaxError, "nonlocal binding not allowed for type parameter '%U'", name); @@ -574,11 +574,11 @@ analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, if (PySet_Discard(global, name) < 0) return 0; if (flags & DEF_TYPE_PARAM) { - if (PySet_Add(typeparams, name) < 0) + if (PySet_Add(type_params, name) < 0) return 0; } else { - if (PySet_Discard(typeparams, name) < 0) + if (PySet_Discard(type_params, name) < 0) return 0; } return 1; @@ -871,12 +871,12 @@ update_symbols(PyObject *symbols, PyObject *scopes, static int analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free, - PyObject *global, PyObject *typeparams, + PyObject *global, PyObject *type_params, PySTEntryObject *class_entry, PyObject **child_free); static int analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, - PyObject *global, PyObject *typeparams, + PyObject *global, PyObject *type_params, PySTEntryObject *class_entry) { PyObject *name, *v, *local = NULL, *scopes = NULL, *newbound = NULL; @@ -939,7 +939,7 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, while (PyDict_Next(ste->ste_symbols, &pos, &name, &v)) { long flags = PyLong_AS_LONG(v); if (!analyze_name(ste, scopes, name, flags, - bound, local, free, global, typeparams, class_entry)) + bound, local, free, global, type_params, class_entry)) goto error; } @@ -1002,7 +1002,7 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, !entry->ste_generator; if (!analyze_child_block(entry, newbound, newfree, newglobal, - typeparams, new_class_entry, &child_free)) + type_params, new_class_entry, &child_free)) { goto error; } @@ -1066,11 +1066,11 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, static int analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free, - PyObject *global, PyObject *typeparams, + PyObject *global, PyObject *type_params, PySTEntryObject *class_entry, PyObject** child_free) { PyObject *temp_bound = NULL, *temp_global = NULL, *temp_free = NULL; - PyObject *temp_typeparams = NULL; + PyObject *temp_type_params = NULL; /* Copy the bound/global/free sets. @@ -1088,30 +1088,30 @@ analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free, temp_global = PySet_New(global); if (!temp_global) goto error; - temp_typeparams = PySet_New(typeparams); - if (!temp_typeparams) + temp_type_params = PySet_New(type_params); + if (!temp_type_params) goto error; if (!analyze_block(entry, temp_bound, temp_free, temp_global, - temp_typeparams, class_entry)) + temp_type_params, class_entry)) goto error; *child_free = temp_free; Py_DECREF(temp_bound); Py_DECREF(temp_global); - Py_DECREF(temp_typeparams); + Py_DECREF(temp_type_params); return 1; error: Py_XDECREF(temp_bound); Py_XDECREF(temp_free); Py_XDECREF(temp_global); - Py_XDECREF(temp_typeparams); + Py_XDECREF(temp_type_params); return 0; } static int symtable_analyze(struct symtable *st) { - PyObject *free, *global, *typeparams; + PyObject *free, *global, *type_params; int r; free = PySet_New(NULL); @@ -1122,16 +1122,16 @@ symtable_analyze(struct symtable *st) Py_DECREF(free); return 0; } - typeparams = PySet_New(NULL); - if (!typeparams) { + type_params = PySet_New(NULL); + if (!type_params) { Py_DECREF(free); Py_DECREF(global); return 0; } - r = analyze_block(st->st_top, NULL, free, global, typeparams, NULL); + r = analyze_block(st->st_top, NULL, free, global, type_params, NULL); Py_DECREF(free); Py_DECREF(global); - Py_DECREF(typeparams); + Py_DECREF(type_params); return r; } @@ -1313,7 +1313,7 @@ symtable_add_def(struct symtable *st, PyObject *name, int flag, } static int -symtable_enter_typeparam_block(struct symtable *st, identifier name, +symtable_enter_type_param_block(struct symtable *st, identifier name, void *ast, int has_defaults, int has_kwdefaults, enum _stmt_kind kind, int lineno, int col_offset, @@ -1472,10 +1472,10 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_SEQ_WITH_NULL(st, expr, s->v.FunctionDef.args->kw_defaults); if (s->v.FunctionDef.decorator_list) VISIT_SEQ(st, expr, s->v.FunctionDef.decorator_list); - if (asdl_seq_LEN(s->v.FunctionDef.typeparams) > 0) { - if (!symtable_enter_typeparam_block( + if (asdl_seq_LEN(s->v.FunctionDef.type_params) > 0) { + if (!symtable_enter_type_param_block( st, s->v.FunctionDef.name, - (void *)s->v.FunctionDef.typeparams, + (void *)s->v.FunctionDef.type_params, s->v.FunctionDef.args->defaults != NULL, has_kwonlydefaults(s->v.FunctionDef.args->kwonlyargs, s->v.FunctionDef.args->kw_defaults), @@ -1483,7 +1483,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) LOCATION(s))) { VISIT_QUIT(st, 0); } - VISIT_SEQ(st, typeparam, s->v.FunctionDef.typeparams); + VISIT_SEQ(st, type_param, s->v.FunctionDef.type_params); } if (!symtable_visit_annotations(st, s, s->v.FunctionDef.args, s->v.FunctionDef.returns)) @@ -1496,7 +1496,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_SEQ(st, stmt, s->v.FunctionDef.body); if (!symtable_exit_block(st)) VISIT_QUIT(st, 0); - if (asdl_seq_LEN(s->v.FunctionDef.typeparams) > 0) { + if (asdl_seq_LEN(s->v.FunctionDef.type_params) > 0) { if (!symtable_exit_block(st)) VISIT_QUIT(st, 0); } @@ -1507,14 +1507,14 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_QUIT(st, 0); if (s->v.ClassDef.decorator_list) VISIT_SEQ(st, expr, s->v.ClassDef.decorator_list); - if (asdl_seq_LEN(s->v.ClassDef.typeparams) > 0) { - if (!symtable_enter_typeparam_block(st, s->v.ClassDef.name, - (void *)s->v.ClassDef.typeparams, + if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) { + if (!symtable_enter_type_param_block(st, s->v.ClassDef.name, + (void *)s->v.ClassDef.type_params, false, false, s->kind, LOCATION(s))) { VISIT_QUIT(st, 0); } - VISIT_SEQ(st, typeparam, s->v.ClassDef.typeparams); + VISIT_SEQ(st, type_param, s->v.ClassDef.type_params); } VISIT_SEQ(st, expr, s->v.ClassDef.bases); VISIT_SEQ(st, keyword, s->v.ClassDef.keywords); @@ -1524,7 +1524,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_QUIT(st, 0); tmp = st->st_private; st->st_private = s->v.ClassDef.name; - if (asdl_seq_LEN(s->v.ClassDef.typeparams) > 0) { + if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) { if (!symtable_add_def(st, &_Py_ID(__type_params__), DEF_LOCAL, LOCATION(s))) { VISIT_QUIT(st, 0); @@ -1539,7 +1539,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) st->st_private = tmp; if (!symtable_exit_block(st)) VISIT_QUIT(st, 0); - if (asdl_seq_LEN(s->v.ClassDef.typeparams) > 0) { + if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) { if (!symtable_exit_block(st)) VISIT_QUIT(st, 0); } @@ -1550,16 +1550,16 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) assert(s->v.TypeAlias.name->kind == Name_kind); PyObject *name = s->v.TypeAlias.name->v.Name.id; int is_in_class = st->st_cur->ste_type == ClassBlock; - int is_generic = asdl_seq_LEN(s->v.TypeAlias.typeparams) > 0; + int is_generic = asdl_seq_LEN(s->v.TypeAlias.type_params) > 0; if (is_generic) { - if (!symtable_enter_typeparam_block( + if (!symtable_enter_type_param_block( st, name, - (void *)s->v.TypeAlias.typeparams, + (void *)s->v.TypeAlias.type_params, false, false, s->kind, LOCATION(s))) { VISIT_QUIT(st, 0); } - VISIT_SEQ(st, typeparam, s->v.TypeAlias.typeparams); + VISIT_SEQ(st, type_param, s->v.TypeAlias.type_params); } if (!symtable_enter_block(st, name, TypeAliasBlock, (void *)s, LOCATION(s))) @@ -1785,10 +1785,10 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) s->v.AsyncFunctionDef.args->kw_defaults); if (s->v.AsyncFunctionDef.decorator_list) VISIT_SEQ(st, expr, s->v.AsyncFunctionDef.decorator_list); - if (asdl_seq_LEN(s->v.AsyncFunctionDef.typeparams) > 0) { - if (!symtable_enter_typeparam_block( + if (asdl_seq_LEN(s->v.AsyncFunctionDef.type_params) > 0) { + if (!symtable_enter_type_param_block( st, s->v.AsyncFunctionDef.name, - (void *)s->v.AsyncFunctionDef.typeparams, + (void *)s->v.AsyncFunctionDef.type_params, s->v.AsyncFunctionDef.args->defaults != NULL, has_kwonlydefaults(s->v.AsyncFunctionDef.args->kwonlyargs, s->v.AsyncFunctionDef.args->kw_defaults), @@ -1796,7 +1796,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) LOCATION(s))) { VISIT_QUIT(st, 0); } - VISIT_SEQ(st, typeparam, s->v.AsyncFunctionDef.typeparams); + VISIT_SEQ(st, type_param, s->v.AsyncFunctionDef.type_params); } if (!symtable_visit_annotations(st, s, s->v.AsyncFunctionDef.args, s->v.AsyncFunctionDef.returns)) @@ -1811,7 +1811,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_SEQ(st, stmt, s->v.AsyncFunctionDef.body); if (!symtable_exit_block(st)) VISIT_QUIT(st, 0); - if (asdl_seq_LEN(s->v.AsyncFunctionDef.typeparams) > 0) { + if (asdl_seq_LEN(s->v.AsyncFunctionDef.type_params) > 0) { if (!symtable_exit_block(st)) VISIT_QUIT(st, 0); } @@ -2110,7 +2110,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e) } static int -symtable_visit_typeparam(struct symtable *st, typeparam_ty tp) +symtable_visit_type_param(struct symtable *st, type_param_ty tp) { if (++st->recursion_depth > st->recursion_limit) { PyErr_SetString(PyExc_RecursionError, From 13e460086b007691f2ca1c5ff677cdb70d19eba8 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Mon, 22 May 2023 02:33:50 -0400 Subject: [PATCH 15/47] gh-99834: Update macOS installer to Tcl/Tk 8.6.13. (GH-104738) --- Mac/BuildScript/build-installer.py | 6 +++--- .../macOS/2023-05-21-23-54-52.gh-issue-99834.6ANPts.rst | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/macOS/2023-05-21-23-54-52.gh-issue-99834.6ANPts.rst diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 5474f87fd6cb62..177fc2044f694b 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -264,10 +264,10 @@ def library_recipes(): tk_patches = ['tk868_on_10_8_10_9.patch'] else: - tcl_tk_ver='8.6.12' - tcl_checksum='87ea890821d2221f2ab5157bc5eb885f' + tcl_tk_ver='8.6.13' + tcl_checksum='43a1fae7412f61ff11de2cfd05d28cfc3a73762f354a417c62370a54e2caf066' - tk_checksum='1d6dcf6120356e3d211e056dff5e462a' + tk_checksum='2e65fa069a23365440a3c56c556b8673b5e32a283800d8d9b257e3f584ce0675' tk_patches = [ ] diff --git a/Misc/NEWS.d/next/macOS/2023-05-21-23-54-52.gh-issue-99834.6ANPts.rst b/Misc/NEWS.d/next/macOS/2023-05-21-23-54-52.gh-issue-99834.6ANPts.rst new file mode 100644 index 00000000000000..aeb64d25b09add --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2023-05-21-23-54-52.gh-issue-99834.6ANPts.rst @@ -0,0 +1 @@ +Update macOS installer to Tcl/Tk 8.6.13. From 6fba0314765d7c58a33e9859d9cd5bcc35d2fd0a Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Mon, 22 May 2023 02:43:43 -0400 Subject: [PATCH 16/47] Update macOS installer ReadMe and Welcome screens for 3.12.0b1. (GH-104739) --- Mac/BuildScript/resources/ReadMe.rtf | 4 ++-- Mac/BuildScript/resources/Welcome.rtf | 29 ++++----------------------- 2 files changed, 6 insertions(+), 27 deletions(-) diff --git a/Mac/BuildScript/resources/ReadMe.rtf b/Mac/BuildScript/resources/ReadMe.rtf index dbe8c520a7c78d..5bc356d5267045 100644 --- a/Mac/BuildScript/resources/ReadMe.rtf +++ b/Mac/BuildScript/resources/ReadMe.rtf @@ -1,4 +1,4 @@ -{\rtf1\ansi\ansicpg1252\cocoartf2639 +{\rtf1\ansi\ansicpg1252\cocoartf2709 \cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fswiss\fcharset0 Helvetica-Bold;\f2\fswiss\fcharset0 Helvetica-Oblique; \f3\fmodern\fcharset0 CourierNewPSMT;\f4\fmodern\fcharset0 Courier;} {\colortbl;\red255\green255\blue255;} @@ -11,7 +11,7 @@ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\partightenfactor0 \f1\b \cf0 NOTE: -\f0\b0 This is an alpha preview of Python 3.12.0, the next feature release of Python 3. It is not intended for production use.\ +\f0\b0 This is a beta preview of Python 3.12.0, the next feature release of Python 3. It is not intended for production use.\ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 \cf0 \ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 diff --git a/Mac/BuildScript/resources/Welcome.rtf b/Mac/BuildScript/resources/Welcome.rtf index 1f109ee9f13e0d..83b7aa9d883a16 100644 --- a/Mac/BuildScript/resources/Welcome.rtf +++ b/Mac/BuildScript/resources/Welcome.rtf @@ -1,4 +1,4 @@ -{\rtf1\ansi\ansicpg1252\cocoartf2708 +{\rtf1\ansi\ansicpg1252\cocoartf2709 \cocoascreenfonts1\cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fswiss\fcharset0 Helvetica-Bold;\f2\fmodern\fcharset0 CourierNewPSMT; } {\colortbl;\red255\green255\blue255;} @@ -12,8 +12,9 @@ \f1\b macOS $MACOSX_DEPLOYMENT_TARGET \f0\b0 .\ \ +\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\partightenfactor0 -\f1\b Python for macOS +\f1\b \cf0 Python for macOS \f0\b0 consists of the {\field{\*\fldinst{HYPERLINK "https://www.python.org"}}{\fldrslt Python}} programming language interpreter and its batteries-included standard library to allow easy access to macOS features. It also includes the Python integrated development environment, \f1\b IDLE \f0\b0 . You can also use the included @@ -25,28 +26,6 @@ At the end of this install, click on \f0 to install a set of current SSL root certificates.\ \ -\f1\b macOS 13 Ventura users -\f0\b0 : Due to an issue with the macOS -\f1\b Installer -\f0\b0 app, installation of some third-party packages including this Python package may fail with a vague -\f1\b "The installer encountered an error" -\f0\b0 message if the -\f1\b Installer -\f0\b0 app does not have permission to access the folder containing the downloaded installer file, typically in the -\f1\b Downloads -\f0\b0 folder. Go to -\f1\b System Settings -\f0\b0 -> -\f1\b Privacy & Security -\f0\b0 -> -\f1\b Files and Folders -\f0\b0 , then click the mark in front of -\f1\b Installer -\f0\b0 to expand, and enable -\f1\b Downloads Folder -\f0\b0 by moving the toggle to the right. See {\field{\*\fldinst{HYPERLINK "https://github.com/python/cpython/issues/103207"}}{\fldrslt https://github.com/python/cpython/issues/103207}} for up-to-date information on this issue.\ -\ - \f1\b NOTE: -\f0\b0 This is an alpha test preview of Python 3.12.0, the next feature release of Python 3. It is not intended for production use.\ +\f0\b0 This is a beta test preview of Python 3.12.0, the next feature release of Python 3. It is not intended for production use.\ } \ No newline at end of file From 9bc80dac47f6d43d0bbfbf10c4cc3848b175e97f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 22 May 2023 11:54:41 +0300 Subject: [PATCH 17/47] gh-94473: Flatten arguments in tkinter.Canvas.coords() (GH-98479) It now accepts not only "x1, y1, x2, y2, ..." and "[x1, y1, x2, y2, ...]", but also "(x1, y1), (x2, y2), ..." and "[(x1, y1), (x2, y2), ...]". --- Doc/whatsnew/3.12.rst | 11 +++++++++++ Lib/test/test_tkinter/test_widgets.py | 6 ++++++ Lib/tkinter/__init__.py | 2 +- .../2022-10-20-14-03-58.gh-issue-94473.pzGX73.rst | 3 +++ 4 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2022-10-20-14-03-58.gh-issue-94473.pzGX73.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index caf21078b9bd59..efbd2ca3de122a 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -532,6 +532,17 @@ threading profiling functions in all running threads in addition to the calling one. (Contributed by Pablo Galindo in :gh:`93503`.) +tkinter +------- + +* ``tkinter.Canvas.coords()`` now flattens its arguments. + It now accepts not only coordinates as separate arguments + (``x1, y1, x2, y2, ...``) and a sequence of coordinates + (``[x1, y1, x2, y2, ...]``), but also coordinates grouped in pairs + (``(x1, y1), (x2, y2), ...`` and ``[(x1, y1), (x2, y2), ...]``), + like ``create_*()`` methods. + (Contributed by Serhiy Storchaka in :gh:`94473`.) + types ----- diff --git a/Lib/test/test_tkinter/test_widgets.py b/Lib/test/test_tkinter/test_widgets.py index b45245162f2446..76cc16e5b977de 100644 --- a/Lib/test/test_tkinter/test_widgets.py +++ b/Lib/test/test_tkinter/test_widgets.py @@ -901,6 +901,12 @@ def test_coords(self): c.coords(i, [21, 31, 41, 51, 61, 11]) self.assertEqual(c.coords(i), [21.0, 31.0, 41.0, 51.0, 61.0, 11.0]) + c.coords(i, (22, 32), (42, 52), (62, 12)) + self.assertEqual(c.coords(i), [22.0, 32.0, 42.0, 52.0, 62.0, 12.0]) + + c.coords(i, [(23, 33), (43, 53), (63, 13)]) + self.assertEqual(c.coords(i), [23.0, 33.0, 43.0, 53.0, 63.0, 13.0]) + c.coords(i, 20, 30, 60, 10) self.assertEqual(c.coords(i), [20.0, 30.0, 60.0, 10.0]) self.assertEqual(c.bbox(i), (18, 8, 62, 32)) diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index bf0b3b92155938..c675c511e04533 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -2817,7 +2817,7 @@ def canvasy(self, screeny, gridspacing=None): def coords(self, *args): """Return a list of coordinates for the item given in ARGS.""" - # XXX Should use _flatten on args + args = _flatten(args) return [self.tk.getdouble(x) for x in self.tk.splitlist( self.tk.call((self._w, 'coords') + args))] diff --git a/Misc/NEWS.d/next/Library/2022-10-20-14-03-58.gh-issue-94473.pzGX73.rst b/Misc/NEWS.d/next/Library/2022-10-20-14-03-58.gh-issue-94473.pzGX73.rst new file mode 100644 index 00000000000000..f70722a6f0f69c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-20-14-03-58.gh-issue-94473.pzGX73.rst @@ -0,0 +1,3 @@ +Flatten arguments in :meth:`tkinter.Canvas.coords`. It now accepts not only +``x1, y1, x2, y2, ...`` and ``[x1, y1, x2, y2, ...]``, but also ``(x1, y1), +(x2, y2), ...`` and ``[(x1, y1), (x2, y2), ...]``. From 0a7796052acb9cec8b13f8d0a5f304f56f26ec5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristi=C3=A1n=20Maureira-Fredes?= Date: Mon, 22 May 2023 12:30:07 +0200 Subject: [PATCH 18/47] gh-102856: Allow comments inside multi-line f-string expresions (#104006) --- Lib/test/test_fstring.py | 41 +++++++++++++++++++++++++++++++++++++--- Parser/tokenizer.c | 4 ---- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index fcb12d25ff9dd6..3ba2f943e6e968 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -661,15 +661,50 @@ def test_comments(self): self.assertEqual(f'{"#"}', '#') self.assertEqual(f'{d["#"]}', 'hash') - self.assertAllRaise(SyntaxError, "f-string expression part cannot include '#'", - ["f'{1#}'", # error because the expression becomes "(1#)" - "f'{3(#)}'", + self.assertAllRaise(SyntaxError, "'{' was never closed", + ["f'{1#}'", # error because everything after '#' is a comment "f'{#}'", + "f'one: {1#}'", + "f'{1# one} {2 this is a comment still#}'", ]) self.assertAllRaise(SyntaxError, r"f-string: unmatched '\)'", ["f'{)#}'", # When wrapped in parens, this becomes # '()#)'. Make sure that doesn't compile. ]) + self.assertEqual(f'''A complex trick: { +2 # two +}''', 'A complex trick: 2') + self.assertEqual(f''' +{ +40 # fourty ++ # plus +2 # two +}''', '\n42') + self.assertEqual(f''' +{ +40 # fourty ++ # plus +2 # two +}''', '\n42') + + self.assertEqual(f''' +# this is not a comment +{ # the following operation it's +3 # this is a number +* 2}''', '\n# this is not a comment\n6') + self.assertEqual(f''' +{# f'a {comment}' +86 # constant +# nothing more +}''', '\n86') + + self.assertAllRaise(SyntaxError, r"f-string: valid expression required before '}'", + ["""f''' +{ +# only a comment +}''' +""", # this is equivalent to f'{}' + ]) def test_many_expressions(self): # Create a string with many expressions in it. Note that diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index fc4afccbfc4008..472d4174726354 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1818,10 +1818,6 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t /* Skip comment, unless it's a type comment */ if (c == '#') { - if (INSIDE_FSTRING(tok)) { - return MAKE_TOKEN(syntaxerror(tok, "f-string expression part cannot include '#'")); - } - const char* p = NULL; const char *prefix, *type_start; int current_starting_col_offset; From 729b252241966f464cc46e176fb854dbcc5296cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marta=20G=C3=B3mez=20Mac=C3=ADas?= Date: Mon, 22 May 2023 13:30:18 +0200 Subject: [PATCH 19/47] gh-104741: Add line number attribute to indentation error exception (#104743) --- Lib/test/test_tabnanny.py | 2 +- Lib/test/test_tokenize.py | 11 ++++++++++- Python/Python-tokenize.c | 15 +++++++++------ 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_tabnanny.py b/Lib/test/test_tabnanny.py index dac47318011d9d..aa700118f735d9 100644 --- a/Lib/test/test_tabnanny.py +++ b/Lib/test/test_tabnanny.py @@ -317,7 +317,7 @@ def test_with_errored_file(self): with TemporaryPyFile(SOURCE_CODES["wrong_indented"]) as file_path: stderr = f"{file_path!r}: Token Error: " stderr += ('unindent does not match any outer indentation level' - ' (, line 3)') + ' (, line 3)') self.validate_cmd(file_path, stderr=stderr, expect_failure=True) def test_with_error_free_file(self): diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index dda7243bfa19fe..8e7ab3d4b7b578 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -92,9 +92,18 @@ def k(x): readline = BytesIO(indent_error_file).readline with self.assertRaisesRegex(IndentationError, "unindent does not match any " - "outer indentation level"): + "outer indentation level") as e: for tok in tokenize(readline): pass + self.assertEqual(e.exception.lineno, 3) + self.assertEqual(e.exception.filename, '') + self.assertEqual(e.exception.end_lineno, None) + self.assertEqual(e.exception.end_offset, None) + self.assertEqual( + e.exception.msg, + 'unindent does not match any outer indentation level') + self.assertEqual(e.exception.offset, 9) + self.assertEqual(e.exception.text, ' x += 5\n') def test_int(self): # Ordinary integers and binary operators diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 43b44be94583ee..f7e32d3af9a9f7 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -89,11 +89,9 @@ _tokenizer_error(struct tok_state *tok) } return -1; case E_DEDENT: - PyErr_Format(PyExc_IndentationError, - "unindent does not match any outer indentation level " - "(, line %d)", - tok->lineno); - return -1; + msg = "unindent does not match any outer indentation level"; + errtype = PyExc_IndentationError; + break; case E_INTR: if (!PyErr_Occurred()) { PyErr_SetNone(PyExc_KeyboardInterrupt); @@ -131,7 +129,12 @@ _tokenizer_error(struct tok_state *tok) goto exit; } - tmp = Py_BuildValue("(OnnOii)", tok->filename, tok->lineno, 0, error_line, 0, 0); + Py_ssize_t offset = _PyPegen_byte_offset_to_character_offset(error_line, tok->inp - tok->buf); + if (offset == -1) { + result = -1; + goto exit; + } + tmp = Py_BuildValue("(OnnOOO)", tok->filename, tok->lineno, offset, error_line, Py_None, Py_None); if (!tmp) { result = -1; goto exit; From 5360cb3d5608ab375de6cd8c0b408459f3fa953a Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Mon, 22 May 2023 04:45:20 -0700 Subject: [PATCH 20/47] gh-101282: Apply BOLT optimizations to libpython for shared builds (#104709) Apply BOLT optimizations to libpython for shared builds. Most of the C code is in libpython so it is critical to apply BOLT there fully realize BOLT benefits. This change also reworks how BOLT instrumentation is applied. It effectively removes the readelf based logic added in gh-101525 and replaces it with a mechanism that saves a copy of the pre-bolt binary and restores that copy when necessary. This allows us to perform BOLT optimizations without having to manually delete the output binary to force a new bolt run. Also: - add a clean-bolt target for purging BOLT files and hook that up to the clean target - .gitignore BOLT related files Before and after this refactor, `make` will no-op after a previous run. Both versions should also share common make DAG deficiencies where targets fail to trigger as often as they need to or can trigger prematurely in certain scenarios. e.g. after this change you may need to `rm profile-bolt-stamp` to force a BOLT run because there aren't appropriate non-phony targets for BOLT's make target to depend on. To make it easier to iterate on custom BOLT settings, the flags to pass to instrumentation and application are now defined in configure and can be overridden by passing BOLT_INSTRUMENT_FLAGS and BOLT_APPLY_FLAGS. --- .gitignore | 5 + Doc/using/configure.rst | 7 + Makefile.pre.in | 65 ++++++-- ...-05-20-16-09-59.gh-issue-101282.FvRARb.rst | 4 + configure | 147 +++++------------- configure.ac | 55 ++++++- 6 files changed, 153 insertions(+), 130 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2023-05-20-16-09-59.gh-issue-101282.FvRARb.rst diff --git a/.gitignore b/.gitignore index d9c4a7972f076d..ef7642b09bc5d2 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,10 @@ *.gc?? *.profclang? *.profraw +# Copies of binaries before BOLT optimizations. +*.prebolt +# BOLT profile data. +*.fdata *.dyn .gdb_history .purify @@ -124,6 +128,7 @@ Tools/unicode/data/ /platform /profile-clean-stamp /profile-run-stamp +/profile-bolt-stamp /Python/deepfreeze/*.c /pybuilddir.txt /pyconfig.h diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index ce858ab2c8d79e..fbe280d6413170 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -314,6 +314,13 @@ also be used to improve performance. is dependent on a combination of the build environment + the other optimization configure args + the CPU architecture, and not all combinations are supported. + BOLT versions before LLVM 16 are known to crash BOLT under some scenarios. + Use of LLVM 16 or newer for BOLT optimization is strongly encouraged. + + The :envvar:`!BOLT_INSTRUMENT_FLAGS` and :envvar:`!BOLT_APPLY_FLAGS` + :program:`configure` variables can be defined to override the default set of + arguments for :program:`llvm-bolt` to instrument and apply BOLT data to + binaries, respectively. .. versionadded:: 3.12 diff --git a/Makefile.pre.in b/Makefile.pre.in index da3a8f6c13f90b..eb79c9c4ca1801 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -672,21 +672,55 @@ profile-opt: profile-run-stamp -rm -f profile-clean-stamp $(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS_NODIST) $(PGO_PROF_USE_FLAG)" LDFLAGS_NODIST="$(LDFLAGS_NODIST)" -.PHONY: bolt-opt -bolt-opt: @PREBOLT_RULE@ +# List of binaries that BOLT runs on. +BOLT_BINARIES := @BOLT_BINARIES@ + +BOLT_INSTRUMENT_FLAGS := @BOLT_INSTRUMENT_FLAGS@ +BOLT_APPLY_FLAGS := @BOLT_APPLY_FLAGS@ + +.PHONY: clean-bolt +clean-bolt: + # Profile data. rm -f *.fdata - @if $(READELF) -p .note.bolt_info $(BUILDPYTHON) | grep BOLT > /dev/null; then\ - echo "skip: $(BUILDPYTHON) is already BOLTed."; \ - else \ - @LLVM_BOLT@ ./$(BUILDPYTHON) -instrument -instrumentation-file-append-pid -instrumentation-file=$(abspath $(BUILDPYTHON).bolt) -o $(BUILDPYTHON).bolt_inst; \ - ./$(BUILDPYTHON).bolt_inst $(PROFILE_TASK) || true; \ - @MERGE_FDATA@ $(BUILDPYTHON).*.fdata > $(BUILDPYTHON).fdata; \ - @LLVM_BOLT@ ./$(BUILDPYTHON) -o $(BUILDPYTHON).bolt -data=$(BUILDPYTHON).fdata -update-debug-sections -reorder-blocks=ext-tsp -reorder-functions=hfsort+ -split-functions -icf=1 -inline-all -split-eh -reorder-functions-use-hot-size -peepholes=none -jump-tables=aggressive -inline-ap -indirect-call-promotion=all -dyno-stats -use-gnu-stack -frame-opt=hot; \ - rm -f *.fdata; \ - rm -f $(BUILDPYTHON).bolt_inst; \ - mv $(BUILDPYTHON).bolt $(BUILDPYTHON); \ - fi + # Pristine binaries before BOLT optimization. + rm -f *.prebolt + # BOLT instrumented binaries. + rm -f *.bolt_inst + +profile-bolt-stamp: $(BUILDPYTHON) + # Ensure a pristine, pre-BOLT copy of the binary and no profile data from last run. + for bin in $(BOLT_BINARIES); do \ + prebolt="$${bin}.prebolt"; \ + if [ -e "$${prebolt}" ]; then \ + echo "Restoring pre-BOLT binary $${prebolt}"; \ + mv "$${bin}.prebolt" "$${bin}"; \ + fi; \ + cp "$${bin}" "$${prebolt}"; \ + rm -f $${bin}.bolt.*.fdata $${bin}.fdata; \ + done + # Instrument each binary. + for bin in $(BOLT_BINARIES); do \ + @LLVM_BOLT@ "$${bin}" -instrument -instrumentation-file-append-pid -instrumentation-file=$(abspath $${bin}.bolt) -o $${bin}.bolt_inst $(BOLT_INSTRUMENT_FLAGS); \ + mv "$${bin}.bolt_inst" "$${bin}"; \ + done + # Run instrumented binaries to collect data. + $(RUNSHARED) ./$(BUILDPYTHON) $(PROFILE_TASK) || true + # Merge all the data files together. + for bin in $(BOLT_BINARIES); do \ + @MERGE_FDATA@ $${bin}.*.fdata > "$${bin}.fdata"; \ + rm -f $${bin}.*.fdata; \ + done + # Run bolt against the merged data to produce an optimized binary. + for bin in $(BOLT_BINARIES); do \ + @LLVM_BOLT@ "$${bin}.prebolt" -o "$${bin}.bolt" -data="$${bin}.fdata" $(BOLT_APPLY_FLAGS); \ + mv "$${bin}.bolt" "$${bin}"; \ + done + touch $@ +.PHONY: bolt-opt +bolt-opt: + $(MAKE) @PREBOLT_RULE@ + $(MAKE) profile-bolt-stamp # Compile and run with gcov .PHONY: coverage @@ -2623,10 +2657,11 @@ profile-removal: rm -f $(COVERAGE_INFO) rm -rf $(COVERAGE_REPORT) rm -f profile-run-stamp + rm -f profile-bolt-stamp .PHONY: clean -clean: clean-retain-profile - @if test @DEF_MAKE_ALL_RULE@ = profile-opt; then \ +clean: clean-retain-profile clean-bolt + @if test @DEF_MAKE_ALL_RULE@ = profile-opt -o @DEF_MAKE_ALL_RULE@ = bolt-opt; then \ rm -f profile-gen-stamp profile-clean-stamp; \ $(MAKE) profile-removal; \ fi diff --git a/Misc/NEWS.d/next/Build/2023-05-20-16-09-59.gh-issue-101282.FvRARb.rst b/Misc/NEWS.d/next/Build/2023-05-20-16-09-59.gh-issue-101282.FvRARb.rst new file mode 100644 index 00000000000000..cc70d47c6c16dd --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-05-20-16-09-59.gh-issue-101282.FvRARb.rst @@ -0,0 +1,4 @@ +BOLT optimization is now applied to the libpython shared library if building +a shared library. BOLT instrumentation and application settings can now be +influenced via the ``BOLT_INSTRUMENT_FLAGS`` and ``BOLT_APPLY_FLAGS`` +configure variables. diff --git a/configure b/configure index 7aad4fe89e3cbf..2b863be108be26 100755 --- a/configure +++ b/configure @@ -883,10 +883,11 @@ CFLAGS_NODIST BASECFLAGS CFLAGS_ALIASING OPT +BOLT_APPLY_FLAGS +BOLT_INSTRUMENT_FLAGS +BOLT_BINARIES MERGE_FDATA LLVM_BOLT -ac_ct_READELF -READELF PREBOLT_RULE LLVM_PROF_FOUND LLVM_PROFDATA @@ -1105,6 +1106,8 @@ CPPFLAGS CPP HOSTRUNNER PROFILE_TASK +BOLT_INSTRUMENT_FLAGS +BOLT_APPLY_FLAGS LIBUUID_CFLAGS LIBUUID_LIBS LIBFFI_CFLAGS @@ -1916,6 +1919,10 @@ Some influential environment variables: HOSTRUNNER Program to run CPython for the host platform PROFILE_TASK Python args for PGO generation task + BOLT_INSTRUMENT_FLAGS + Arguments to llvm-bolt when instrumenting binaries + BOLT_APPLY_FLAGS + Arguments to llvm-bolt when creating a BOLT optimized binary LIBUUID_CFLAGS C compiler flags for LIBUUID, overriding pkg-config LIBUUID_LIBS @@ -8106,112 +8113,6 @@ if test "$Py_BOLT" = 'true' ; then DEF_MAKE_ALL_RULE="bolt-opt" DEF_MAKE_RULE="build_all" - - if test -n "$ac_tool_prefix"; then - for ac_prog in readelf - do - # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args. -set dummy $ac_tool_prefix$ac_prog; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_READELF+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$READELF"; then - ac_cv_prog_READELF="$READELF" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_READELF="$ac_tool_prefix$ac_prog" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -READELF=$ac_cv_prog_READELF -if test -n "$READELF"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $READELF" >&5 -$as_echo "$READELF" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - - test -n "$READELF" && break - done -fi -if test -z "$READELF"; then - ac_ct_READELF=$READELF - for ac_prog in readelf -do - # Extract the first word of "$ac_prog", so it can be a program name with args. -set dummy $ac_prog; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_ac_ct_READELF+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$ac_ct_READELF"; then - ac_cv_prog_ac_ct_READELF="$ac_ct_READELF" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_ac_ct_READELF="$ac_prog" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -ac_ct_READELF=$ac_cv_prog_ac_ct_READELF -if test -n "$ac_ct_READELF"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_READELF" >&5 -$as_echo "$ac_ct_READELF" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - - test -n "$ac_ct_READELF" && break -done - - if test "x$ac_ct_READELF" = x; then - READELF=""notfound"" - else - case $cross_compiling:$ac_tool_warned in -yes:) -{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 -$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} -ac_tool_warned=yes ;; -esac - READELF=$ac_ct_READELF - fi -fi - - if test "$READELF" == "notfound" - then - as_fn_error $? "readelf is required for a --enable-bolt build but could not be found." "$LINENO" 5 - fi - # -fno-reorder-blocks-and-partition is required for bolt to work. # Possibly GCC only. { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -fno-reorder-blocks-and-partition" >&5 @@ -8474,6 +8375,36 @@ $as_echo "\"Found merge-fdata\"" >&6; } fi fi + +BOLT_BINARIES='$(BUILDPYTHON)' +if test "x$enable_shared" = xyes; then : + + BOLT_BINARIES="${BOLT_BINARIES} \$(INSTSONAME)" + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking BOLT_INSTRUMENT_FLAGS" >&5 +$as_echo_n "checking BOLT_INSTRUMENT_FLAGS... " >&6; } +if test -z "${BOLT_INSTRUMENT_FLAGS}" +then + BOLT_INSTRUMENT_FLAGS= +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $BOLT_INSTRUMENT_FLAGS" >&5 +$as_echo "$BOLT_INSTRUMENT_FLAGS" >&6; } + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking BOLT_APPLY_FLAGS" >&5 +$as_echo_n "checking BOLT_APPLY_FLAGS... " >&6; } +if test -z "${BOLT_APPLY_FLAGS}" +then + BOLT_APPLY_FLAGS=-update-debug-sections -reorder-blocks=ext-tsp -reorder-functions=hfsort+ -split-functions -icf=1 -inline-all -split-eh -reorder-functions-use-hot-size -peepholes=none -jump-tables=aggressive -inline-ap -indirect-call-promotion=all -dyno-stats -use-gnu-stack -frame-opt=hot + + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $BOLT_APPLY_FLAGS" >&5 +$as_echo "$BOLT_APPLY_FLAGS" >&6; } + # XXX Shouldn't the code above that fiddles with BASECFLAGS and OPT be # merged with this chunk of code? diff --git a/configure.ac b/configure.ac index 115998e0753b26..786d3414eb0e06 100644 --- a/configure.ac +++ b/configure.ac @@ -2028,13 +2028,6 @@ if test "$Py_BOLT" = 'true' ; then DEF_MAKE_ALL_RULE="bolt-opt" DEF_MAKE_RULE="build_all" - AC_SUBST(READELF) - AC_CHECK_TOOLS(READELF, [readelf], "notfound") - if test "$READELF" == "notfound" - then - AC_MSG_ERROR([readelf is required for a --enable-bolt build but could not be found.]) - fi - # -fno-reorder-blocks-and-partition is required for bolt to work. # Possibly GCC only. AX_CHECK_COMPILE_FLAG([-fno-reorder-blocks-and-partition],[ @@ -2067,6 +2060,54 @@ if test "$Py_BOLT" = 'true' ; then fi fi +dnl Enable BOLT of libpython if built. +AC_SUBST(BOLT_BINARIES) +BOLT_BINARIES='$(BUILDPYTHON)' +AS_VAR_IF([enable_shared], [yes], [ + BOLT_BINARIES="${BOLT_BINARIES} \$(INSTSONAME)" +]) + +AC_ARG_VAR( + [BOLT_INSTRUMENT_FLAGS], + [Arguments to llvm-bolt when instrumenting binaries] +) +AC_MSG_CHECKING([BOLT_INSTRUMENT_FLAGS]) +if test -z "${BOLT_INSTRUMENT_FLAGS}" +then + BOLT_INSTRUMENT_FLAGS= +fi +AC_MSG_RESULT([$BOLT_INSTRUMENT_FLAGS]) + +AC_ARG_VAR( + [BOLT_APPLY_FLAGS], + [Arguments to llvm-bolt when creating a BOLT optimized binary] +) +AC_MSG_CHECKING([BOLT_APPLY_FLAGS]) +if test -z "${BOLT_APPLY_FLAGS}" +then + AS_VAR_SET( + [BOLT_APPLY_FLAGS], + [m4_join([ ], + [-update-debug-sections], + [-reorder-blocks=ext-tsp], + [-reorder-functions=hfsort+], + [-split-functions], + [-icf=1], + [-inline-all], + [-split-eh], + [-reorder-functions-use-hot-size], + [-peepholes=none], + [-jump-tables=aggressive], + [-inline-ap], + [-indirect-call-promotion=all], + [-dyno-stats], + [-use-gnu-stack], + [-frame-opt=hot] + )] + ) +fi +AC_MSG_RESULT([$BOLT_APPLY_FLAGS]) + # XXX Shouldn't the code above that fiddles with BASECFLAGS and OPT be # merged with this chunk of code? From 5612078f68e9688fbf3b1d8565abc79b77501000 Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Mon, 22 May 2023 14:01:29 +0200 Subject: [PATCH 21/47] Python 3.12.0b1 --- Doc/tools/extensions/pyspecific.py | 2 +- Include/patchlevel.h | 6 +- Lib/pydoc_data/topics.py | 339 ++- Misc/NEWS.d/3.12.0b1.rst | 2412 +++++++++++++++++ ...2-06-20-15-15-11.gh-issue-90656.kFBbKe.rst | 7 - ...-12-18-07-24-44.gh-issue-100220.BgSV7C.rst | 4 - ...3-02-11-05-31-05.gh-issue-99069.X4LDvY.rst | 1 - ...-04-14-10-24-37.gh-issue-103532.H1djkd.rst | 4 - ...-05-04-10-56-14.gh-issue-104106.-W9BJS.rst | 1 - ...-05-14-19-00-19.gh-issue-104490.1tA4AF.rst | 1 - ...3-05-15-09-34-08.gh-issue-99017.nToOQu.rst | 1 - ...-05-20-16-09-59.gh-issue-101282.FvRARb.rst | 4 - ...2-09-15-15-21-34.gh-issue-96803.ynBKIS.rst | 6 - ...2-10-29-10-13-20.gh-issue-98836.Cy5h_z.rst | 4 - ...-02-09-23-09-29.gh-issue-101408._paFIF.rst | 2 - ...-03-28-12-31-51.gh-issue-103091.CzZyaZ.rst | 1 - ...-04-13-16-54-00.gh-issue-103509.A26Qu8.rst | 5 - ...-04-14-23-05-52.gh-issue-103295.GRHY1Z.rst | 5 - ...-04-24-10-31-59.gh-issue-103743.2xYA1K.rst | 2 - ...-04-28-18-04-38.gh-issue-103968.EnVvOx.rst | 4 - ...-05-02-21-05-54.gh-issue-104109.0tnDZV.rst | 5 - ...-05-18-20-53-05.gh-issue-101291.ZBh9aR.rst | 3 - .../2019-12-01-12-58-31.bpo-31821.1FNmwk.rst | 1 - .../2020-02-11-15-54-40.bpo-39610.fvgsCl.rst | 2 - ...2-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst | 3 - ...2-11-08-12-36-25.gh-issue-99184.KIaqzz.rst | 2 - ...3-01-30-15-40-29.gh-issue-97933.nUlp3r.rst | 2 - ...-03-07-17-37-00.gh-issue-102500.RUSQhz.rst | 3 - ...3-03-25-23-24-38.gh-issue-88691.2SWBd1.rst | 1 - ...-03-31-17-24-03.gh-issue-103082.isRUcV.rst | 1 - ...-04-01-00-46-31.gh-issue-102700.493NB4.rst | 1 - ...3-04-02-22-14-57.gh-issue-84436.hvMgwF.rst | 3 - ...-04-07-12-18-41.gh-issue-103323.9802br.rst | 3 - ...-04-08-17-13-07.gh-issue-103242.ysI1b3.rst | 2 - ...-04-09-04-30-02.gh-issue-103333.gKOetS.rst | 1 - ...3-04-09-22-21-57.gh-issue-77757._Ow-u2.rst | 3 - ...3-04-12-19-55-24.gh-issue-82012.FlcJAh.rst | 5 - ...-04-12-20-18-51.gh-issue-103488.vYvlHD.rst | 3 - ...3-04-12-20-22-03.gh-issue-87729.99A7UO.rst | 4 - ...-04-13-00-58-55.gh-issue-103492.P4k0Ay.rst | 1 - ...-04-14-22-35-23.gh-issue-101517.5EqM-S.rst | 1 - ...-04-16-14-38-39.gh-issue-100530.OR6-sn.rst | 1 - ...-04-17-16-00-32.gh-issue-102856.UunJ7y.rst | 1 - ...-04-20-16-17-51.gh-issue-103650.K1MFXR.rst | 1 - ...-04-21-16-12-41.gh-issue-103590.7DHDOE.rst | 1 - ...-04-21-17-03-14.gh-issue-102310.anLjDx.rst | 1 - ...-04-24-14-38-16.gh-issue-103793.kqoH6Q.rst | 3 - ...-04-24-21-47-38.gh-issue-103801.WaBanq.rst | 1 - ...-04-25-08-43-11.gh-issue-103763.ZLBZk1.rst | 23 - ...-04-25-20-56-01.gh-issue-103845.V7NYFn.rst | 1 - ...-04-26-15-14-23.gh-issue-103899.1pqKPF.rst | 3 - ...-04-26-16-26-35.gh-issue-103907.kiONZQ.rst | 2 - ...-04-26-17-56-18.gh-issue-103895.ESB6tn.rst | 3 - ...-05-01-08-08-05.gh-issue-102213.nfH-4C.rst | 1 - ...-05-01-12-03-52.gh-issue-104018.PFxGS4.rst | 1 - ...-05-01-14-10-38.gh-issue-104028.dxfh13.rst | 2 - ...-05-01-14-48-29.gh-issue-104066.pzoUZQ.rst | 2 - ...-05-01-21-05-47.gh-issue-104078.vRaBsU.rst | 1 - ...-05-02-18-29-49.gh-issue-104142._5Et6I.rst | 2 - ...-05-03-17-46-47.gh-issue-104108.GOxAYt.rst | 6 - ...3-05-05-12-14-47.gh-issue-99113.-RAdnv.rst | 6 - ...3-05-05-13-18-56.gh-issue-99113.hT1ajK.rst | 11 - ...-05-08-10-34-55.gh-issue-104263.ctHWI8.rst | 6 - ...-05-10-20-52-29.gh-issue-103082.y3LG5Q.rst | 5 - ...-05-11-15-56-07.gh-issue-104405.tXV5fn.rst | 2 - ...-05-12-13-30-04.gh-issue-102818.rnv1mH.rst | 5 - ...-05-14-18-56-54.gh-issue-104482.yaQsv8.rst | 1 - ...-05-16-19-17-48.gh-issue-104572.eBZQYS.rst | 2 - ...-05-18-13-00-21.gh-issue-104615.h_rtw2.rst | 2 - ...-05-20-23-08-48.gh-issue-102856.Knv9WT.rst | 1 - ...3-03-07-23-30-29.gh-issue-99202.hhiAJF.rst | 1 - ...3-03-10-04-59-35.gh-issue-86094.zOYdy8.rst | 2 - ...3-04-25-22-58-08.gh-issue-48241.l1Gxxh.rst | 1 - ...-04-26-23-55-31.gh-issue-103629.-0reqn.rst | 2 - ...3-05-14-12-11-28.gh-issue-67056.nVC2Rf.rst | 2 - ...3-04-30-20-01-18.gh-issue-88496.y65vUb.rst | 1 - ...-05-17-15-11-11.gh-issue-104496.wjav-y.rst | 1 - ...-05-17-17-32-21.gh-issue-104499.hNeqV4.rst | 1 - .../2018-07-16-14-10-29.bpo-22708.592iRR.rst | 3 - .../2020-02-25-00-43-22.bpo-39744.hgK689.rst | 1 - .../2020-05-25-12-42-36.bpo-17258.lf2554.rst | 2 - .../2021-05-16-14-28-30.bpo-24964.Oa5Ie_.rst | 3 - .../2021-11-07-15-31-25.bpo-23041.564i32.rst | 2 - .../2021-11-19-23-37-18.bpo-45606.UW5XE1.rst | 5 - .../2021-12-03-23-00-56.bpo-44844.tvg2VY.rst | 1 - .../2022-02-19-14-19-34.bpo-46797.6BXZX4.rst | 4 - ...2-05-02-16-21-05.gh-issue-92184.hneGVW.rst | 3 - ...2-07-03-23-13-28.gh-issue-94518.511Tbh.rst | 1 - ...2-07-06-11-10-37.gh-issue-51574.sveUeD.rst | 2 - ...2-07-16-17-15-29.gh-issue-94906.C4G8DG.rst | 1 - ...2-08-27-21-41-41.gh-issue-87474.9X-kxt.rst | 1 - ...2-09-03-09-24-02.gh-issue-96534.EU4Oxv.rst | 1 - ...2-09-07-09-32-07.gh-issue-96522.t73oqp.rst | 1 - ...2-10-09-14-47-42.gh-issue-98040.IN3qab.rst | 2 - ...2-10-20-14-03-58.gh-issue-94473.pzGX73.rst | 3 - ...2-10-21-16-23-31.gh-issue-97850.N46coo.rst | 2 - ...2-10-21-17-20-57.gh-issue-98040.3btbmA.rst | 1 - ...2-11-10-16-26-47.gh-issue-99353.DQFjnt.rst | 3 - ...3-01-14-17-54-56.gh-issue-95299.vUhpKz.rst | 1 - ...3-01-22-14-53-12.gh-issue-89550.c1U23f.rst | 2 - ...3-02-06-16-45-18.gh-issue-83861.mMbIU3.rst | 4 - ...-02-09-22-24-34.gh-issue-101640.oFuEpB.rst | 1 - ...-02-11-15-01-32.gh-issue-101688.kwXmfM.rst | 2 - ...3-02-11-21-18-10.gh-issue-85984.nvzOD0.rst | 3 - ...3-02-17-21-14-40.gh-issue-78079.z3Szr6.rst | 3 - ...3-02-19-12-37-08.gh-issue-62432.GnBFIB.rst | 3 - ...-02-21-14-57-34.gh-issue-102114.uUDQzb.rst | 1 - ...-03-06-18-49-57.gh-issue-101362.eSSy6L.rst | 2 - ...3-03-08-02-45-46.gh-issue-91896.kgON_a.rst | 1 - ...-03-14-11-20-19.gh-issue-101819.0-h0it.rst | 2 - ...3-03-15-00-37-43.gh-issue-81079.heTAod.rst | 2 - ...3-03-15-12-18-07.gh-issue-97696.DtnpIC.rst | 6 - ...-03-23-15-24-38.gh-issue-102953.YR4KaK.rst | 4 - ...-03-24-20-49-48.gh-issue-103000.6eVNZI.rst | 2 - ...-03-28-09-13-31.gh-issue-103015.ETTfNf.rst | 3 - ...-03-31-01-13-00.gh-issue-103143.6eMluy.rst | 1 - ...-04-01-23-01-31.gh-issue-103176.FBsdxa.rst | 2 - ...-04-02-17-51-08.gh-issue-103193.xrZbM1.rst | 2 - ...3-04-02-22-04-26.gh-issue-75586.526iJm.rst | 1 - ...-04-02-23-05-22.gh-issue-103204.bbDmu0.rst | 3 - ...-04-03-21-08-53.gh-issue-103220.OW_Bj5.rst | 2 - ...-04-03-22-02-35.gh-issue-100479.kNBjQm.rst | 4 - ...-04-03-23-43-12.gh-issue-103092.3xqk4y.rst | 1 - ...-04-03-23-44-34.gh-issue-102978.gy9eVk.rst | 3 - ...3-04-04-12-43-38.gh-issue-93910.jurMzv.rst | 1 - ...-04-04-21-27-51.gh-issue-103092.7s7Bzf.rst | 1 - ...-04-04-21-44-25.gh-issue-103092.Dz0_Xn.rst | 1 - ...-04-05-01-28-53.gh-issue-103225.QD3JVU.rst | 1 - ...-04-06-04-35-59.gh-issue-103285.rCZ9-G.rst | 1 - ...-04-06-16-55-51.gh-issue-102778.BWeAmE.rst | 1 - ...-04-06-17-28-36.gh-issue-103256.1syxfs.rst | 6 - ...3-04-07-15-09-26.gh-issue-74690.0f886b.rst | 3 - ...3-04-07-15-15-40.gh-issue-74690.un84hh.rst | 8 - ...-04-08-00-48-40.gh-issue-103092.5EFts0.rst | 1 - ...-04-08-01-33-12.gh-issue-103357.vjin28.rst | 3 - ...-04-09-06-59-36.gh-issue-103092.vskbro.rst | 1 - ...-04-11-21-38-39.gh-issue-103449.-nxmhb.rst | 1 - ...-04-12-06-00-02.gh-issue-103462.w6yBlM.rst | 4 - ...-04-12-13-04-16.gh-issue-103472.C6bOHv.rst | 2 - ...-04-12-17-59-55.gh-issue-103365.UBEE0U.rst | 1 - ...-04-13-13-17-47.gh-issue-103489.ZSZgmu.rst | 4 - ...-04-13-19-43-15.gh-issue-103525.uY4VYg.rst | 2 - ...-04-14-06-32-54.gh-issue-103533.n_AfcS.rst | 1 - ...-04-14-21-12-32.gh-issue-103538.M4FK_v.rst | 3 - ...-04-14-21-16-05.gh-issue-103548.lagdpp.rst | 4 - ...-04-15-11-21-38.gh-issue-103559.a9rYHG.rst | 1 - ...-04-15-12-19-14.gh-issue-103556.TEf-2m.rst | 3 - ...-04-16-18-29-04.gh-issue-103578.fly1wc.rst | 1 - ...-04-16-19-48-21.gh-issue-103584.3mBTuM.rst | 12 - ...-04-17-14-47-28.gh-issue-103596.ME1y3_.rst | 2 - ...3-04-19-16-08-53.gh-issue-84976.HwbzlD.rst | 5 - ...-04-21-10-25-39.gh-issue-103636.YK6NEa.rst | 1 - ...-04-22-02-41-06.gh-issue-103673.oE7S_k.rst | 2 - ...3-04-22-11-20-27.gh-issue-89415.YHk760.rst | 2 - ...3-04-22-12-30-10.gh-issue-92248.NcVTKR.rst | 2 - ...-04-22-21-34-13.gh-issue-103693.SBtuLQ.rst | 1 - ...-04-22-22-37-39.gh-issue-103699.NizCjc.rst | 2 - ...3-04-23-15-39-17.gh-issue-81403.zVz9Td.rst | 3 - ...-04-24-00-34-23.gh-issue-103685.U14jBM.rst | 1 - ...3-04-24-16-00-28.gh-issue-90750.da0Xi8.rst | 3 - ...-04-24-23-07-56.gh-issue-103791.bBPWdS.rst | 3 - ...-04-25-16-31-00.gh-issue-103839.tpyLhI.rst | 2 - ...-04-25-17-03-18.gh-issue-103857.Mr2Cak.rst | 2 - ...-04-25-19-58-13.gh-issue-103861.JeozgD.rst | 2 - ...3-04-25-22-06-00.gh-issue-74940.TOacQ9.rst | 2 - ...3-04-25-22-59-06.gh-issue-99944.pst8iT.rst | 1 - ...-04-26-09-38-47.gh-issue-103872.8LBsDz.rst | 1 - ...-04-26-09-54-25.gh-issue-103848.aDSnpR.rst | 2 - ...-04-26-15-14-36.gh-issue-103583.iCMDFt.rst | 2 - ...-04-26-18-12-13.gh-issue-103636.-KvCgO.rst | 1 - ...-04-27-00-05-32.gh-issue-102628.X230E-.rst | 2 - ...-04-27-00-45-41.gh-issue-100370.MgZ3KY.rst | 2 - ...3-04-27-18-46-31.gh-issue-68968.E3tnhy.rst | 1 - ...-04-27-20-03-08.gh-issue-103935.Uaf2M0.rst | 1 - ...3-04-28-18-04-23.gh-issue-88773.xXCNJw.rst | 1 - ...-04-28-19-08-50.gh-issue-103977.msF70A.rst | 1 - ...-04-29-18-23-16.gh-issue-103987.sRgALL.rst | 2 - ...-05-01-16-43-28.gh-issue-104035.MrJBw8.rst | 2 - ...-05-01-17-58-28.gh-issue-103963.XWlHx7.rst | 1 - ...-05-01-19-10-05.gh-issue-103629.81bpZz.rst | 1 - ...-05-02-04-49-45.gh-issue-103822.m0QdAO.rst | 1 - ...-05-02-20-43-03.gh-issue-104102.vgSdEJ.rst | 2 - ...-05-02-21-05-30.gh-issue-104104.9tjplT.rst | 2 - ...-05-03-03-14-33.gh-issue-104114.RG26RD.rst | 3 - ...-05-03-16-50-24.gh-issue-104144.yNkjL8.rst | 3 - ...-05-03-16-51-53.gh-issue-104144.653Q0P.rst | 2 - ...3-05-03-19-22-24.gh-issue-90208.tI00da.rst | 4 - ...3-05-05-18-52-22.gh-issue-65772.w5P5Wv.rst | 1 - ...-05-06-20-37-46.gh-issue-102613.QZG9iX.rst | 3 - ...-05-07-19-56-45.gh-issue-104265.fVblry.rst | 4 - ...3-05-08-15-39-00.gh-issue-87695.f6iO7v.rst | 2 - ...-05-08-15-50-59.gh-issue-104310.fXVSPY.rst | 3 - ...-05-08-20-57-17.gh-issue-104307.DSB93G.rst | 1 - ...-05-08-23-01-59.gh-issue-104139.83Tnt-.rst | 3 - ...-05-09-18-46-24.gh-issue-104301.gNnbId.rst | 1 - ...-05-10-19-33-36.gh-issue-103000.j0KSfD.rst | 2 - ...-05-11-01-07-42.gh-issue-102613.uMsokt.rst | 2 - ...-05-11-07-50-00.gh-issue-104392.YSllzt.rst | 2 - ...-05-11-21-32-18.gh-issue-101520.l9MjRE.rst | 2 - ...-05-12-19-29-28.gh-issue-103857.0IzSxr.rst | 1 - ...-05-16-10-07-16.gh-issue-104536.hFWD8f.rst | 3 - ...3-05-16-11-02-44.gh-issue-75367.qLWR35.rst | 1 - ...-05-17-03-14-07.gh-issue-104484.y6KxL6.rst | 1 - ...-05-17-08-01-36.gh-issue-104372.jpoWs6.rst | 1 - ...-05-17-16-58-23.gh-issue-104555.5rb5oM.rst | 7 - ...-05-17-20-03-01.gh-issue-104340.kp_XmX.rst | 1 - ...-05-17-21-01-48.gh-issue-104600.E6CK35.rst | 2 - ...-03-07-20-59-17.gh-issue-102153.14CLSZ.rst | 3 - ...3-04-17-14-38-12.gh-issue-99108.720lG8.rst | 2 - ...-05-01-15-03-25.gh-issue-104049.b01Y3g.rst | 2 - ...3-05-02-17-56-32.gh-issue-99889.l664SU.rst | 2 - ...2-11-06-18-42-38.gh-issue-75729.uGYJrv.rst | 2 - ...-03-17-22-00-47.gh-issue-102795.z21EoC.rst | 1 - ...-04-08-00-50-23.gh-issue-103329.M38tqF.rst | 1 - ...3-04-25-12-19-37.gh-issue-86275.-RoLIt.rst | 2 - ...-05-14-03-00-00.gh-issue-104461.Rmex11.rst | 3 - ...-05-15-02-22-44.gh-issue-104494.Bkrbfn.rst | 2 - ...-05-11-15-12-11.gh-issue-104389.EiOhB3.rst | 2 - ...3-03-18-21-38-00.gh-issue-88013.Z3loxC.rst | 2 - ...-03-24-11-25-28.gh-issue-102997.dredy2.rst | 1 - ...-04-11-09-22-22.gh-issue-103088.6AJEuR.rst | 1 - ...-04-12-10-49-21.gh-issue-103088.Yjj-qJ.rst | 1 - ...3-04-24-15-51-11.gh-issue-82814.GI3UkZ.rst | 3 - ...-05-18-22-46-03.gh-issue-104623.HJZhm1.rst | 1 - ...-03-24-11-20-47.gh-issue-102997.ZgQkbq.rst | 1 - ...-04-04-13-37-28.gh-issue-103207.x0vvQp.rst | 2 - ...3-04-24-18-37-48.gh-issue-60436.in-IyF.rst | 1 - ...-05-04-21-47-59.gh-issue-104180.lEJCwd.rst | 2 - ...-05-18-08-52-04.gh-issue-103545.pi5k2N.rst | 1 - ...-05-18-22-31-49.gh-issue-104623.6h7Xfx.rst | 1 - ...3-05-21-23-54-52.gh-issue-99834.6ANPts.rst | 1 - README.rst | 2 +- 232 files changed, 2669 insertions(+), 628 deletions(-) create mode 100644 Misc/NEWS.d/3.12.0b1.rst delete mode 100644 Misc/NEWS.d/next/Build/2022-06-20-15-15-11.gh-issue-90656.kFBbKe.rst delete mode 100644 Misc/NEWS.d/next/Build/2022-12-18-07-24-44.gh-issue-100220.BgSV7C.rst delete mode 100644 Misc/NEWS.d/next/Build/2023-02-11-05-31-05.gh-issue-99069.X4LDvY.rst delete mode 100644 Misc/NEWS.d/next/Build/2023-04-14-10-24-37.gh-issue-103532.H1djkd.rst delete mode 100644 Misc/NEWS.d/next/Build/2023-05-04-10-56-14.gh-issue-104106.-W9BJS.rst delete mode 100644 Misc/NEWS.d/next/Build/2023-05-14-19-00-19.gh-issue-104490.1tA4AF.rst delete mode 100644 Misc/NEWS.d/next/Build/2023-05-15-09-34-08.gh-issue-99017.nToOQu.rst delete mode 100644 Misc/NEWS.d/next/Build/2023-05-20-16-09-59.gh-issue-101282.FvRARb.rst delete mode 100644 Misc/NEWS.d/next/C API/2022-09-15-15-21-34.gh-issue-96803.ynBKIS.rst delete mode 100644 Misc/NEWS.d/next/C API/2022-10-29-10-13-20.gh-issue-98836.Cy5h_z.rst delete mode 100644 Misc/NEWS.d/next/C API/2023-02-09-23-09-29.gh-issue-101408._paFIF.rst delete mode 100644 Misc/NEWS.d/next/C API/2023-03-28-12-31-51.gh-issue-103091.CzZyaZ.rst delete mode 100644 Misc/NEWS.d/next/C API/2023-04-13-16-54-00.gh-issue-103509.A26Qu8.rst delete mode 100644 Misc/NEWS.d/next/C API/2023-04-14-23-05-52.gh-issue-103295.GRHY1Z.rst delete mode 100644 Misc/NEWS.d/next/C API/2023-04-24-10-31-59.gh-issue-103743.2xYA1K.rst delete mode 100644 Misc/NEWS.d/next/C API/2023-04-28-18-04-38.gh-issue-103968.EnVvOx.rst delete mode 100644 Misc/NEWS.d/next/C API/2023-05-02-21-05-54.gh-issue-104109.0tnDZV.rst delete mode 100644 Misc/NEWS.d/next/C API/2023-05-18-20-53-05.gh-issue-101291.ZBh9aR.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2019-12-01-12-58-31.bpo-31821.1FNmwk.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-02-11-15-54-40.bpo-39610.fvgsCl.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-11-08-12-36-25.gh-issue-99184.KIaqzz.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-01-30-15-40-29.gh-issue-97933.nUlp3r.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-07-17-37-00.gh-issue-102500.RUSQhz.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-25-23-24-38.gh-issue-88691.2SWBd1.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-01-00-46-31.gh-issue-102700.493NB4.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-02-22-14-57.gh-issue-84436.hvMgwF.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-07-12-18-41.gh-issue-103323.9802br.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-08-17-13-07.gh-issue-103242.ysI1b3.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-09-04-30-02.gh-issue-103333.gKOetS.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-09-22-21-57.gh-issue-77757._Ow-u2.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-12-19-55-24.gh-issue-82012.FlcJAh.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-12-20-18-51.gh-issue-103488.vYvlHD.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-12-20-22-03.gh-issue-87729.99A7UO.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-13-00-58-55.gh-issue-103492.P4k0Ay.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-14-22-35-23.gh-issue-101517.5EqM-S.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-16-14-38-39.gh-issue-100530.OR6-sn.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-17-16-00-32.gh-issue-102856.UunJ7y.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-20-16-17-51.gh-issue-103650.K1MFXR.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-21-16-12-41.gh-issue-103590.7DHDOE.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-21-17-03-14.gh-issue-102310.anLjDx.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-24-14-38-16.gh-issue-103793.kqoH6Q.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-24-21-47-38.gh-issue-103801.WaBanq.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-25-08-43-11.gh-issue-103763.ZLBZk1.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-25-20-56-01.gh-issue-103845.V7NYFn.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-26-15-14-23.gh-issue-103899.1pqKPF.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-26-16-26-35.gh-issue-103907.kiONZQ.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-26-17-56-18.gh-issue-103895.ESB6tn.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-01-08-08-05.gh-issue-102213.nfH-4C.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-01-12-03-52.gh-issue-104018.PFxGS4.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-10-38.gh-issue-104028.dxfh13.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-48-29.gh-issue-104066.pzoUZQ.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-01-21-05-47.gh-issue-104078.vRaBsU.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-02-18-29-49.gh-issue-104142._5Et6I.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-03-17-46-47.gh-issue-104108.GOxAYt.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-05-12-14-47.gh-issue-99113.-RAdnv.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-05-13-18-56.gh-issue-99113.hT1ajK.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-08-10-34-55.gh-issue-104263.ctHWI8.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-10-20-52-29.gh-issue-103082.y3LG5Q.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-11-15-56-07.gh-issue-104405.tXV5fn.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-12-13-30-04.gh-issue-102818.rnv1mH.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-14-18-56-54.gh-issue-104482.yaQsv8.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-16-19-17-48.gh-issue-104572.eBZQYS.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-18-13-00-21.gh-issue-104615.h_rtw2.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-20-23-08-48.gh-issue-102856.Knv9WT.rst delete mode 100644 Misc/NEWS.d/next/Documentation/2023-03-07-23-30-29.gh-issue-99202.hhiAJF.rst delete mode 100644 Misc/NEWS.d/next/Documentation/2023-03-10-04-59-35.gh-issue-86094.zOYdy8.rst delete mode 100644 Misc/NEWS.d/next/Documentation/2023-04-25-22-58-08.gh-issue-48241.l1Gxxh.rst delete mode 100644 Misc/NEWS.d/next/Documentation/2023-04-26-23-55-31.gh-issue-103629.-0reqn.rst delete mode 100644 Misc/NEWS.d/next/Documentation/2023-05-14-12-11-28.gh-issue-67056.nVC2Rf.rst delete mode 100644 Misc/NEWS.d/next/IDLE/2023-04-30-20-01-18.gh-issue-88496.y65vUb.rst delete mode 100644 Misc/NEWS.d/next/IDLE/2023-05-17-15-11-11.gh-issue-104496.wjav-y.rst delete mode 100644 Misc/NEWS.d/next/IDLE/2023-05-17-17-32-21.gh-issue-104499.hNeqV4.rst delete mode 100644 Misc/NEWS.d/next/Library/2018-07-16-14-10-29.bpo-22708.592iRR.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-02-25-00-43-22.bpo-39744.hgK689.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-05-25-12-42-36.bpo-17258.lf2554.rst delete mode 100644 Misc/NEWS.d/next/Library/2021-05-16-14-28-30.bpo-24964.Oa5Ie_.rst delete mode 100644 Misc/NEWS.d/next/Library/2021-11-07-15-31-25.bpo-23041.564i32.rst delete mode 100644 Misc/NEWS.d/next/Library/2021-11-19-23-37-18.bpo-45606.UW5XE1.rst delete mode 100644 Misc/NEWS.d/next/Library/2021-12-03-23-00-56.bpo-44844.tvg2VY.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-02-19-14-19-34.bpo-46797.6BXZX4.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-05-02-16-21-05.gh-issue-92184.hneGVW.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-07-03-23-13-28.gh-issue-94518.511Tbh.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-07-06-11-10-37.gh-issue-51574.sveUeD.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-07-16-17-15-29.gh-issue-94906.C4G8DG.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-08-27-21-41-41.gh-issue-87474.9X-kxt.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-09-03-09-24-02.gh-issue-96534.EU4Oxv.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-09-07-09-32-07.gh-issue-96522.t73oqp.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-10-09-14-47-42.gh-issue-98040.IN3qab.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-10-20-14-03-58.gh-issue-94473.pzGX73.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-10-21-16-23-31.gh-issue-97850.N46coo.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-10-21-17-20-57.gh-issue-98040.3btbmA.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-11-10-16-26-47.gh-issue-99353.DQFjnt.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-01-14-17-54-56.gh-issue-95299.vUhpKz.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-01-22-14-53-12.gh-issue-89550.c1U23f.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-06-16-45-18.gh-issue-83861.mMbIU3.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-09-22-24-34.gh-issue-101640.oFuEpB.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-11-15-01-32.gh-issue-101688.kwXmfM.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-11-21-18-10.gh-issue-85984.nvzOD0.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-17-21-14-40.gh-issue-78079.z3Szr6.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-19-12-37-08.gh-issue-62432.GnBFIB.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-21-14-57-34.gh-issue-102114.uUDQzb.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-06-18-49-57.gh-issue-101362.eSSy6L.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-08-02-45-46.gh-issue-91896.kgON_a.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-14-11-20-19.gh-issue-101819.0-h0it.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-15-00-37-43.gh-issue-81079.heTAod.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-15-12-18-07.gh-issue-97696.DtnpIC.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-23-15-24-38.gh-issue-102953.YR4KaK.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-24-20-49-48.gh-issue-103000.6eVNZI.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-28-09-13-31.gh-issue-103015.ETTfNf.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-31-01-13-00.gh-issue-103143.6eMluy.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-01-23-01-31.gh-issue-103176.FBsdxa.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-02-17-51-08.gh-issue-103193.xrZbM1.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-02-22-04-26.gh-issue-75586.526iJm.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-02-23-05-22.gh-issue-103204.bbDmu0.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-03-21-08-53.gh-issue-103220.OW_Bj5.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-03-22-02-35.gh-issue-100479.kNBjQm.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-03-23-43-12.gh-issue-103092.3xqk4y.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-03-23-44-34.gh-issue-102978.gy9eVk.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-04-12-43-38.gh-issue-93910.jurMzv.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-04-21-27-51.gh-issue-103092.7s7Bzf.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-04-21-44-25.gh-issue-103092.Dz0_Xn.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-05-01-28-53.gh-issue-103225.QD3JVU.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-06-04-35-59.gh-issue-103285.rCZ9-G.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-06-16-55-51.gh-issue-102778.BWeAmE.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-06-17-28-36.gh-issue-103256.1syxfs.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-07-15-09-26.gh-issue-74690.0f886b.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-07-15-15-40.gh-issue-74690.un84hh.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-08-00-48-40.gh-issue-103092.5EFts0.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-08-01-33-12.gh-issue-103357.vjin28.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-09-06-59-36.gh-issue-103092.vskbro.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-11-21-38-39.gh-issue-103449.-nxmhb.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-12-06-00-02.gh-issue-103462.w6yBlM.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-12-13-04-16.gh-issue-103472.C6bOHv.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-12-17-59-55.gh-issue-103365.UBEE0U.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-13-13-17-47.gh-issue-103489.ZSZgmu.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-13-19-43-15.gh-issue-103525.uY4VYg.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-14-06-32-54.gh-issue-103533.n_AfcS.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-14-21-12-32.gh-issue-103538.M4FK_v.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-14-21-16-05.gh-issue-103548.lagdpp.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-15-11-21-38.gh-issue-103559.a9rYHG.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-15-12-19-14.gh-issue-103556.TEf-2m.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-16-18-29-04.gh-issue-103578.fly1wc.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-16-19-48-21.gh-issue-103584.3mBTuM.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-17-14-47-28.gh-issue-103596.ME1y3_.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-19-16-08-53.gh-issue-84976.HwbzlD.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-21-10-25-39.gh-issue-103636.YK6NEa.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-22-02-41-06.gh-issue-103673.oE7S_k.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-22-11-20-27.gh-issue-89415.YHk760.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-22-12-30-10.gh-issue-92248.NcVTKR.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-22-21-34-13.gh-issue-103693.SBtuLQ.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-22-22-37-39.gh-issue-103699.NizCjc.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-23-15-39-17.gh-issue-81403.zVz9Td.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-24-00-34-23.gh-issue-103685.U14jBM.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-24-16-00-28.gh-issue-90750.da0Xi8.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-24-23-07-56.gh-issue-103791.bBPWdS.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-25-16-31-00.gh-issue-103839.tpyLhI.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-25-17-03-18.gh-issue-103857.Mr2Cak.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-25-19-58-13.gh-issue-103861.JeozgD.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-25-22-06-00.gh-issue-74940.TOacQ9.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-25-22-59-06.gh-issue-99944.pst8iT.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-26-09-38-47.gh-issue-103872.8LBsDz.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-26-09-54-25.gh-issue-103848.aDSnpR.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-26-15-14-36.gh-issue-103583.iCMDFt.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-26-18-12-13.gh-issue-103636.-KvCgO.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-27-00-05-32.gh-issue-102628.X230E-.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-27-00-45-41.gh-issue-100370.MgZ3KY.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-27-18-46-31.gh-issue-68968.E3tnhy.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-27-20-03-08.gh-issue-103935.Uaf2M0.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-28-18-04-23.gh-issue-88773.xXCNJw.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-28-19-08-50.gh-issue-103977.msF70A.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-01-16-43-28.gh-issue-104035.MrJBw8.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-01-17-58-28.gh-issue-103963.XWlHx7.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-01-19-10-05.gh-issue-103629.81bpZz.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-02-04-49-45.gh-issue-103822.m0QdAO.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-02-20-43-03.gh-issue-104102.vgSdEJ.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-02-21-05-30.gh-issue-104104.9tjplT.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-03-03-14-33.gh-issue-104114.RG26RD.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-03-16-50-24.gh-issue-104144.yNkjL8.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-03-16-51-53.gh-issue-104144.653Q0P.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-03-19-22-24.gh-issue-90208.tI00da.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-05-18-52-22.gh-issue-65772.w5P5Wv.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-06-20-37-46.gh-issue-102613.QZG9iX.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-07-19-56-45.gh-issue-104265.fVblry.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-08-15-39-00.gh-issue-87695.f6iO7v.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-08-15-50-59.gh-issue-104310.fXVSPY.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-08-20-57-17.gh-issue-104307.DSB93G.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-08-23-01-59.gh-issue-104139.83Tnt-.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-09-18-46-24.gh-issue-104301.gNnbId.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-10-19-33-36.gh-issue-103000.j0KSfD.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-11-01-07-42.gh-issue-102613.uMsokt.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-11-07-50-00.gh-issue-104392.YSllzt.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-11-21-32-18.gh-issue-101520.l9MjRE.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-12-19-29-28.gh-issue-103857.0IzSxr.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-16-10-07-16.gh-issue-104536.hFWD8f.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-16-11-02-44.gh-issue-75367.qLWR35.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-17-03-14-07.gh-issue-104484.y6KxL6.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-17-08-01-36.gh-issue-104372.jpoWs6.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-17-16-58-23.gh-issue-104555.5rb5oM.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-17-20-03-01.gh-issue-104340.kp_XmX.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-05-17-21-01-48.gh-issue-104600.E6CK35.rst delete mode 100644 Misc/NEWS.d/next/Security/2023-03-07-20-59-17.gh-issue-102153.14CLSZ.rst delete mode 100644 Misc/NEWS.d/next/Security/2023-04-17-14-38-12.gh-issue-99108.720lG8.rst delete mode 100644 Misc/NEWS.d/next/Security/2023-05-01-15-03-25.gh-issue-104049.b01Y3g.rst delete mode 100644 Misc/NEWS.d/next/Security/2023-05-02-17-56-32.gh-issue-99889.l664SU.rst delete mode 100644 Misc/NEWS.d/next/Tests/2022-11-06-18-42-38.gh-issue-75729.uGYJrv.rst delete mode 100644 Misc/NEWS.d/next/Tests/2023-03-17-22-00-47.gh-issue-102795.z21EoC.rst delete mode 100644 Misc/NEWS.d/next/Tests/2023-04-08-00-50-23.gh-issue-103329.M38tqF.rst delete mode 100644 Misc/NEWS.d/next/Tests/2023-04-25-12-19-37.gh-issue-86275.-RoLIt.rst delete mode 100644 Misc/NEWS.d/next/Tests/2023-05-14-03-00-00.gh-issue-104461.Rmex11.rst delete mode 100644 Misc/NEWS.d/next/Tests/2023-05-15-02-22-44.gh-issue-104494.Bkrbfn.rst delete mode 100644 Misc/NEWS.d/next/Tools-Demos/2023-05-11-15-12-11.gh-issue-104389.EiOhB3.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-03-18-21-38-00.gh-issue-88013.Z3loxC.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-03-24-11-25-28.gh-issue-102997.dredy2.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-04-11-09-22-22.gh-issue-103088.6AJEuR.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-04-12-10-49-21.gh-issue-103088.Yjj-qJ.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-04-24-15-51-11.gh-issue-82814.GI3UkZ.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-05-18-22-46-03.gh-issue-104623.HJZhm1.rst delete mode 100644 Misc/NEWS.d/next/macOS/2023-03-24-11-20-47.gh-issue-102997.ZgQkbq.rst delete mode 100644 Misc/NEWS.d/next/macOS/2023-04-04-13-37-28.gh-issue-103207.x0vvQp.rst delete mode 100644 Misc/NEWS.d/next/macOS/2023-04-24-18-37-48.gh-issue-60436.in-IyF.rst delete mode 100644 Misc/NEWS.d/next/macOS/2023-05-04-21-47-59.gh-issue-104180.lEJCwd.rst delete mode 100644 Misc/NEWS.d/next/macOS/2023-05-18-08-52-04.gh-issue-103545.pi5k2N.rst delete mode 100644 Misc/NEWS.d/next/macOS/2023-05-18-22-31-49.gh-issue-104623.6h7Xfx.rst delete mode 100644 Misc/NEWS.d/next/macOS/2023-05-21-23-54-52.gh-issue-99834.6ANPts.rst diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index cd8d9febb0d13b..3a5b26f7779618 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -41,7 +41,7 @@ ISSUE_URI = 'https://bugs.python.org/issue?@action=redirect&bpo=%s' GH_ISSUE_URI = 'https://github.com/python/cpython/issues/%s' -SOURCE_URI = 'https://github.com/python/cpython/tree/main/%s' +SOURCE_URI = 'https://github.com/python/cpython/tree/3.12/%s' # monkey-patch reST parser to disable alphabetic and roman enumerated lists from docutils.parsers.rst.states import Body diff --git a/Include/patchlevel.h b/Include/patchlevel.h index aaedd563a905e9..3de9ed9296a47b 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -19,11 +19,11 @@ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 12 #define PY_MICRO_VERSION 0 -#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 7 +#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_BETA +#define PY_RELEASE_SERIAL 1 /* Version as a string */ -#define PY_VERSION "3.12.0a7+" +#define PY_VERSION "3.12.0b1" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 3aaaee67fa35a6..cb742992a48e8f 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Tue Apr 4 17:52:21 2023 +# Autogenerated by Sphinx on Mon May 22 14:02:15 2023 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' @@ -2573,9 +2573,12 @@ '\n' 'Any remaining exceptions that were not handled by any "except*" ' 'clause\n' - 'are re-raised at the end, combined into an exception group along ' - 'with\n' - 'all exceptions that were raised from within "except*" clauses.\n' + 'are re-raised at the end, along with all exceptions that were ' + 'raised\n' + 'from within the "except*" clauses. If this list contains more ' + 'than one\n' + 'exception to reraise, they are combined into an exception ' + 'group.\n' '\n' 'If the raised exception is not an exception group and its type ' 'matches\n' @@ -4587,8 +4590,7 @@ 'case\n' ' performance of a dict insertion, O(n^2) complexity. ' 'See\n' - ' http://www.ocert.org/advisories/ocert-2011-003.html ' - 'for\n' + ' http://ocert.org/advisories/ocert-2011-003.html for\n' ' details.Changing hash values affects the iteration ' 'order of sets.\n' ' Python has never made guarantees about this ordering ' @@ -4651,20 +4653,41 @@ 'traces of\n' ' Python programs.\n' '\n' - 'The debugger’s prompt is "(Pdb)". Typical usage to run a program ' - 'under\n' - 'control of the debugger is:\n' + 'The typical usage to break into the debugger is to insert:\n' '\n' - ' >>> import pdb\n' - ' >>> import mymodule\n' - " >>> pdb.run('mymodule.test()')\n" - ' > (0)?()\n' - ' (Pdb) continue\n' - ' > (1)?()\n' + ' import pdb; pdb.set_trace()\n' + '\n' + 'Or:\n' + '\n' + ' breakpoint()\n' + '\n' + 'at the location you want to break into the debugger, and then ' + 'run the\n' + 'program. You can then step through the code following this ' + 'statement,\n' + 'and continue running without the debugger using the "continue"\n' + 'command.\n' + '\n' + 'New in version 3.7: The built-in "breakpoint()", when called ' + 'with\n' + 'defaults, can be used instead of "import pdb; pdb.set_trace()".\n' + '\n' + ' def double(x):\n' + ' breakpoint()\n' + ' return x * 2\n' + ' val = 3\n' + ' print(f"{val} * 2 is {double(val)}")\n' + '\n' + 'The debugger’s prompt is "(Pdb)", which is the indicator that ' + 'you are\n' + 'in debug mode:\n' + '\n' + ' > ...(3)double()\n' + ' -> return x * 2\n' + ' (Pdb) p x\n' + ' 3\n' ' (Pdb) continue\n' - " NameError: 'spam'\n" - ' > (1)?()\n' - ' (Pdb)\n' + ' 3 * 2 is 6\n' '\n' 'Changed in version 3.3: Tab-completion via the "readline" module ' 'is\n' @@ -4672,13 +4695,12 @@ 'global\n' 'and local names are offered as arguments of the "p" command.\n' '\n' - '"pdb.py" can also be invoked as a script to debug other ' - 'scripts. For\n' - 'example:\n' + 'You can also invoke "pdb" from the command line to debug other\n' + 'scripts. For example:\n' '\n' ' python -m pdb myscript.py\n' '\n' - 'When invoked as a script, pdb will automatically enter ' + 'When invoked as a module, pdb will automatically enter ' 'post-mortem\n' 'debugging if the program being debugged exits abnormally. After ' 'post-\n' @@ -4690,47 +4712,43 @@ 'the\n' 'debugger upon program’s exit.\n' '\n' - 'New in version 3.2: "pdb.py" now accepts a "-c" option that ' - 'executes\n' - 'commands as if given in a ".pdbrc" file, see Debugger Commands.\n' + 'New in version 3.2: "-c" option is introduced to execute ' + 'commands as\n' + 'if given in a ".pdbrc" file, see Debugger Commands.\n' '\n' - 'New in version 3.7: "pdb.py" now accepts a "-m" option that ' - 'execute\n' - 'modules similar to the way "python -m" does. As with a script, ' - 'the\n' - 'debugger will pause execution just before the first line of the\n' - 'module.\n' - '\n' - 'The typical usage to break into the debugger is to insert:\n' + 'New in version 3.7: "-m" option is introduced to execute ' + 'modules\n' + 'similar to the way "python -m" does. As with a script, the ' + 'debugger\n' + 'will pause execution just before the first line of the module.\n' '\n' - ' import pdb; pdb.set_trace()\n' + 'Typical usage to execute a statement under control of the ' + 'debugger is:\n' '\n' - 'at the location you want to break into the debugger, and then ' - 'run the\n' - 'program. You can then step through the code following this ' - 'statement,\n' - 'and continue running without the debugger using the "continue"\n' - 'command.\n' - '\n' - 'New in version 3.7: The built-in "breakpoint()", when called ' - 'with\n' - 'defaults, can be used instead of "import pdb; pdb.set_trace()".\n' + ' >>> import pdb\n' + ' >>> def f(x):\n' + ' ... print(1 / x)\n' + ' >>> pdb.run("f(2)")\n' + ' > (1)()\n' + ' (Pdb) continue\n' + ' 0.5\n' + ' >>>\n' '\n' 'The typical usage to inspect a crashed program is:\n' '\n' ' >>> import pdb\n' - ' >>> import mymodule\n' - ' >>> mymodule.test()\n' + ' >>> def f(x):\n' + ' ... print(1 / x)\n' + ' ...\n' + ' >>> f(0)\n' ' Traceback (most recent call last):\n' ' File "", line 1, in \n' - ' File "./mymodule.py", line 4, in test\n' - ' test2()\n' - ' File "./mymodule.py", line 3, in test2\n' - ' print(spam)\n' - ' NameError: spam\n' + ' File "", line 2, in f\n' + ' ZeroDivisionError: division by zero\n' ' >>> pdb.pm()\n' - ' > ./mymodule.py(3)test2()\n' - ' -> print(spam)\n' + ' > (2)f()\n' + ' (Pdb) p x\n' + ' 0\n' ' (Pdb)\n' '\n' 'The module defines the following functions; each enters the ' @@ -4914,6 +4932,29 @@ 'implicit\n' 'string concatenation "\';\'\';\'" or "";"";"".\n' '\n' + 'To set a temporary global variable, use a *convenience ' + 'variable*. A\n' + '*convenience variable* is a variable whose name starts with ' + '"$". For\n' + 'example, "$foo = 1" sets a global variable "$foo" which you can ' + 'use in\n' + 'the debugger session. The *convenience variables* are cleared ' + 'when\n' + 'the program resumes execution so it’s less likely to interfere ' + 'with\n' + 'your program compared to using normal variables like "foo = 1".\n' + '\n' + 'There are three preset *convenience variables*:\n' + '\n' + '* "$_frame": the current frame you are debugging\n' + '\n' + '* "$_retval": the return value if the frame is returning\n' + '\n' + '* "$_exception": the exception if the frame is raising an ' + 'exception\n' + '\n' + 'New in version 3.12.\n' + '\n' 'If a file ".pdbrc" exists in the user’s home directory or in ' 'the\n' 'current directory, it is read with "\'utf-8\'" encoding and ' @@ -4949,9 +4990,9 @@ '\n' ' Print a stack trace, with the most recent frame at the ' 'bottom. An\n' - ' arrow indicates the current frame, which determines the ' - 'context of\n' - ' most commands.\n' + ' arrow (">") indicates the current frame, which determines ' + 'the\n' + ' context of most commands.\n' '\n' 'd(own) [count]\n' '\n' @@ -5007,7 +5048,7 @@ 'first\n' ' ask confirmation).\n' '\n' - 'disable [bpnumber ...]\n' + 'disable bpnumber [bpnumber ...]\n' '\n' ' Disable the breakpoints given as a space separated list of\n' ' breakpoint numbers. Disabling a breakpoint means it cannot ' @@ -5016,7 +5057,7 @@ 'breakpoint, it\n' ' remains in the list of breakpoints and can be (re-)enabled.\n' '\n' - 'enable [bpnumber ...]\n' + 'enable bpnumber [bpnumber ...]\n' '\n' ' Enable the breakpoints specified.\n' '\n' @@ -5179,7 +5220,9 @@ '\n' 'a(rgs)\n' '\n' - ' Print the argument list of the current function.\n' + ' Print the arguments of the current function and their ' + 'current\n' + ' values.\n' '\n' 'p expression\n' '\n' @@ -5217,6 +5260,54 @@ 'current\n' ' frame.\n' '\n' + ' Note:\n' + '\n' + ' Display evaluates *expression* and compares to the result ' + 'of the\n' + ' previous evaluation of *expression*, so when the result is\n' + ' mutable, display may not be able to pick up the changes.\n' + '\n' + ' Example:\n' + '\n' + ' lst = []\n' + ' breakpoint()\n' + ' pass\n' + ' lst.append(1)\n' + ' print(lst)\n' + '\n' + ' Display won’t realize "lst" has been changed because the ' + 'result of\n' + ' evaluation is modified in place by "lst.append(1)" before ' + 'being\n' + ' compared:\n' + '\n' + ' > example.py(3)()\n' + ' -> pass\n' + ' (Pdb) display lst\n' + ' display lst: []\n' + ' (Pdb) n\n' + ' > example.py(4)()\n' + ' -> lst.append(1)\n' + ' (Pdb) n\n' + ' > example.py(5)()\n' + ' -> print(lst)\n' + ' (Pdb)\n' + '\n' + ' You can do some tricks with copy mechanism to make it work:\n' + '\n' + ' > example.py(3)()\n' + ' -> pass\n' + ' (Pdb) display lst[:]\n' + ' display lst[:]: []\n' + ' (Pdb) n\n' + ' > example.py(4)()\n' + ' -> lst.append(1)\n' + ' (Pdb) n\n' + ' > example.py(5)()\n' + ' -> print(lst)\n' + ' display lst[:]: [1] [old: []]\n' + ' (Pdb)\n' + '\n' ' New in version 3.2.\n' '\n' 'undisplay [expression]\n' @@ -5283,14 +5374,14 @@ 'current\n' ' stack frame. The exclamation point can be omitted unless the ' 'first\n' - ' word of the statement resembles a debugger command, e.g.:' + ' word of the statement resembles a debugger command, e.g.:\n' '\n' ' (Pdb) ! n=42\n' ' (Pdb)\n' '\n' - ' To set a global variable, you can prefix the assignment command ' - ' with \n' - ' a "global" statement on the same line, e.g.:\n' + ' To set a global variable, you can prefix the assignment ' + 'command\n' + ' with a "global" statement on the same line, e.g.:\n' '\n' " (Pdb) global list_options; list_options = ['-l']\n" ' (Pdb)\n' @@ -5321,7 +5412,8 @@ '\n' 'retval\n' '\n' - ' Print the return value for the last return of a function.\n' + ' Print the return value for the last return of the current ' + 'function.\n' '\n' '-[ Footnotes ]-\n' '\n' @@ -9509,8 +9601,7 @@ ' by carefully chosen inputs that exploit the worst case\n' ' performance of a dict insertion, O(n^2) complexity. ' 'See\n' - ' http://www.ocert.org/advisories/ocert-2011-003.html ' - 'for\n' + ' http://ocert.org/advisories/ocert-2011-003.html for\n' ' details.Changing hash values affects the iteration ' 'order of sets.\n' ' Python has never made guarantees about this ordering ' @@ -10164,20 +10255,37 @@ 'Resolving MRO entries\n' '---------------------\n' '\n' - 'If a base that appears in class definition is not an ' + 'object.__mro_entries__(self, bases)\n' + '\n' + ' If a base that appears in a class definition is not an ' 'instance of\n' - '"type", then an "__mro_entries__" method is searched on it. ' - 'If found,\n' - 'it is called with the original bases tuple. This method must ' - 'return a\n' - 'tuple of classes that will be used instead of this base. The ' - 'tuple may\n' - 'be empty, in such case the original base is ignored.\n' + ' "type", then an "__mro_entries__()" method is searched on ' + 'the base.\n' + ' If an "__mro_entries__()" method is found, the base is ' + 'substituted\n' + ' with the result of a call to "__mro_entries__()" when ' + 'creating the\n' + ' class. The method is called with the original bases tuple ' + 'passed to\n' + ' the *bases* parameter, and must return a tuple of classes ' + 'that will\n' + ' be used instead of the base. The returned tuple may be ' + 'empty: in\n' + ' these cases, the original base is ignored.\n' '\n' 'See also:\n' '\n' - ' **PEP 560** - Core support for typing module and generic ' - 'types\n' + ' "types.resolve_bases()"\n' + ' Dynamically resolve bases that are not instances of ' + '"type".\n' + '\n' + ' "types.get_original_bases()"\n' + ' Retrieve a class’s “original bases” prior to ' + 'modifications by\n' + ' "__mro_entries__()".\n' + '\n' + ' **PEP 560**\n' + ' Core support for typing module and generic types.\n' '\n' '\n' 'Determining the appropriate metaclass\n' @@ -11153,6 +11261,61 @@ ' The specification for the Python "match" statement.\n' '\n' '\n' + 'Emulating buffer types\n' + '======================\n' + '\n' + 'The buffer protocol provides a way for Python objects to ' + 'expose\n' + 'efficient access to a low-level memory array. This protocol ' + 'is\n' + 'implemented by builtin types such as "bytes" and ' + '"memoryview", and\n' + 'third-party libraries may define additional buffer types.\n' + '\n' + 'While buffer types are usually implemented in C, it is also ' + 'possible\n' + 'to implement the protocol in Python.\n' + '\n' + 'object.__buffer__(self, flags)\n' + '\n' + ' Called when a buffer is requested from *self* (for ' + 'example, by the\n' + ' "memoryview" constructor). The *flags* argument is an ' + 'integer\n' + ' representing the kind of buffer requested, affecting for ' + 'example\n' + ' whether the returned buffer is read-only or writable.\n' + ' "inspect.BufferFlags" provides a convenient way to ' + 'interpret the\n' + ' flags. The method must return a "memoryview" object.\n' + '\n' + 'object.__release_buffer__(self, buffer)\n' + '\n' + ' Called when a buffer is no longer needed. The *buffer* ' + 'argument is\n' + ' a "memoryview" object that was previously returned by\n' + ' "__buffer__()". The method must release any resources ' + 'associated\n' + ' with the buffer. This method should return "None". Buffer ' + 'objects\n' + ' that do not need to perform any cleanup are not required ' + 'to\n' + ' implement this method.\n' + '\n' + 'New in version 3.12.\n' + '\n' + 'See also:\n' + '\n' + ' **PEP 688** - Making the buffer protocol accessible in ' + 'Python\n' + ' Introduces the Python "__buffer__" and ' + '"__release_buffer__"\n' + ' methods.\n' + '\n' + ' "collections.abc.Buffer"\n' + ' ABC for buffer types.\n' + '\n' + '\n' 'Special method lookup\n' '=====================\n' '\n' @@ -11300,8 +11463,8 @@ ' "casefold()" converts it to ""ss"".\n' '\n' ' The casefolding algorithm is described in section 3.13 ' - 'of the\n' - ' Unicode Standard.\n' + '‘Default\n' + ' Case Folding’ of the Unicode Standard.\n' '\n' ' New in version 3.3.\n' '\n' @@ -11519,8 +11682,9 @@ ' being one of “Lm”, “Lt”, “Lu”, “Ll”, or “Lo”. Note ' 'that this is\n' ' different from the Alphabetic property defined in the ' - 'Unicode\n' - ' Standard.\n' + 'section 4.10\n' + ' ‘Letters, Alphabetic, and Ideographic’ of the Unicode ' + 'Standard.\n' '\n' 'str.isascii()\n' '\n' @@ -11692,8 +11856,8 @@ ' converted to lowercase.\n' '\n' ' The lowercasing algorithm used is described in section ' - '3.13 of the\n' - ' Unicode Standard.\n' + '3.13\n' + ' ‘Default Case Folding’ of the Unicode Standard.\n' '\n' 'str.lstrip([chars])\n' '\n' @@ -12159,8 +12323,8 @@ ' uppercase), but e.g. “Lt” (Letter, titlecase).\n' '\n' ' The uppercasing algorithm used is described in section ' - '3.13 of the\n' - ' Unicode Standard.\n' + '3.13\n' + ' ‘Default Case Folding’ of the Unicode Standard.\n' '\n' 'str.zfill(width)\n' '\n' @@ -12704,9 +12868,10 @@ '\n' 'Any remaining exceptions that were not handled by any "except*" ' 'clause\n' - 'are re-raised at the end, combined into an exception group along ' - 'with\n' - 'all exceptions that were raised from within "except*" clauses.\n' + 'are re-raised at the end, along with all exceptions that were raised\n' + 'from within the "except*" clauses. If this list contains more than ' + 'one\n' + 'exception to reraise, they are combined into an exception group.\n' '\n' 'If the raised exception is not an exception group and its type ' 'matches\n' diff --git a/Misc/NEWS.d/3.12.0b1.rst b/Misc/NEWS.d/3.12.0b1.rst new file mode 100644 index 00000000000000..eb42b45726f5ed --- /dev/null +++ b/Misc/NEWS.d/3.12.0b1.rst @@ -0,0 +1,2412 @@ +.. date: 2023-05-02-17-56-32 +.. gh-issue: 99889 +.. nonce: l664SU +.. release date: 2023-05-22 +.. section: Security + +Fixed a security in flaw in :func:`uu.decode` that could allow for directory +traversal based on the input if no ``out_file`` was specified. + +.. + +.. date: 2023-05-01-15-03-25 +.. gh-issue: 104049 +.. nonce: b01Y3g +.. section: Security + +Do not expose the local on-disk location in directory indexes produced by +:class:`http.client.SimpleHTTPRequestHandler`. + +.. + +.. date: 2023-04-17-14-38-12 +.. gh-issue: 99108 +.. nonce: 720lG8 +.. section: Security + +Upgrade built-in :mod:`hashlib` SHA3 implementation to a verified +implementation from the ``HACL*`` project. Used when OpenSSL is not present +or lacks SHA3. + +.. + +.. date: 2023-03-07-20-59-17 +.. gh-issue: 102153 +.. nonce: 14CLSZ +.. section: Security + +:func:`urllib.parse.urlsplit` now strips leading C0 control and space +characters following the specification for URLs defined by WHATWG in +response to CVE-2023-24329. Patch by Illia Volochii. + +.. + +.. date: 2023-05-20-23-08-48 +.. gh-issue: 102856 +.. nonce: Knv9WT +.. section: Core and Builtins + +Implement PEP 701 changes in the :mod:`tokenize` module. Patch by Marta +Gómez Macías and Pablo Galindo Salgado + +.. + +.. date: 2023-05-18-13-00-21 +.. gh-issue: 104615 +.. nonce: h_rtw2 +.. section: Core and Builtins + +Fix wrong ordering of assignments in code like ``a, a = x, y``. Contributed +by Carl Meyer. + +.. + +.. date: 2023-05-16-19-17-48 +.. gh-issue: 104572 +.. nonce: eBZQYS +.. section: Core and Builtins + +Improve syntax error message for invalid constructs in :pep:`695` contexts +and in annotations when ``from __future__ import annotations`` is active. + +.. + +.. date: 2023-05-14-18-56-54 +.. gh-issue: 104482 +.. nonce: yaQsv8 +.. section: Core and Builtins + +Fix three error handling bugs in ast.c's validation of pattern matching +statements. + +.. + +.. date: 2023-05-12-13-30-04 +.. gh-issue: 102818 +.. nonce: rnv1mH +.. section: Core and Builtins + +Do not add a frame to the traceback in the ``sys.setprofile`` and +``sys.settrace`` trampoline functions. This ensures that frames are not +duplicated if an exception is raised in the callback function, and ensures +that frames are not omitted if a C callback is used and that does not add +the frame. + +.. + +.. date: 2023-05-11-15-56-07 +.. gh-issue: 104405 +.. nonce: tXV5fn +.. section: Core and Builtins + +Fix an issue where some :term:`bytecode` instructions could ignore +:pep:`523` when "inlining" calls. + +.. + +.. date: 2023-05-10-20-52-29 +.. gh-issue: 103082 +.. nonce: y3LG5Q +.. section: Core and Builtins + +Change behavior of ``sys.monitoring.events.LINE`` events in +``sys.monitoring``: Line events now occur when a new line is reached +dynamically, instead of using a static approximation, as before. This makes +the behavior very similar to that of "line" events in ``sys.settrace``. This +should ease porting of tools from 3.11 to 3.12. + +.. + +.. date: 2023-05-08-10-34-55 +.. gh-issue: 104263 +.. nonce: ctHWI8 +.. section: Core and Builtins + +Fix ``float("nan")`` to produce a quiet NaN on platforms (like MIPS) where +the meaning of the signalling / quiet bit is inverted from its usual +meaning. Also introduce a new macro ``Py_INFINITY`` matching C99's +``INFINITY``, and refactor internals to rely on C99's ``NAN`` and +``INFINITY`` macros instead of hard-coding bit patterns for infinities and +NaNs. Thanks Sebastian Berg. + +.. + +.. date: 2023-05-05-13-18-56 +.. gh-issue: 99113 +.. nonce: hT1ajK +.. section: Core and Builtins + +Multi-phase init extension modules may now indicate that they support +running in subinterpreters that have their own GIL. This is done by using +``Py_MOD_PER_INTERPRETER_GIL_SUPPORTED`` as the value for the +``Py_mod_multiple_interpreters`` module def slot. Otherwise the module, by +default, cannot be imported in such subinterpreters. (This does not affect +the main interpreter or subinterpreters that do not have their own GIL.) In +addition to the isolation that multi-phase init already normally requires, +support for per-interpreter GIL involves one additional constraint: +thread-safety. If the module has external (linked) dependencies and those +libraries have any state that isn't thread-safe then the module must do the +additional work to add thread-safety. This should be an uncommon case. + +.. + +.. date: 2023-05-05-12-14-47 +.. gh-issue: 99113 +.. nonce: -RAdnv +.. section: Core and Builtins + +The GIL is now (optionally) per-interpreter. This is the fundamental change +for PEP 684. This is all made possible by virtue of the isolated state of +each interpreter in the process. The behavior of the main interpreter +remains unchanged. Likewise, interpreters created using +``Py_NewInterpreter()`` are not affected. To get an interpreter with its +own GIL, call ``Py_NewInterpreterFromConfig()``. + +.. + +.. date: 2023-05-03-17-46-47 +.. gh-issue: 104108 +.. nonce: GOxAYt +.. section: Core and Builtins + +Multi-phase init extension modules may now indicate whether or not they +actually support multiple interpreters. By default such modules are +expected to support use in multiple interpreters. In the uncommon case that +one does not, it may use the new ``Py_mod_multiple_interpreters`` module def +slot. A value of ``0`` means the module does not support them. ``1`` means +it does. The default is ``1``. + +.. + +.. date: 2023-05-02-18-29-49 +.. gh-issue: 104142 +.. nonce: _5Et6I +.. section: Core and Builtins + +Fix an issue where :class:`list` or :class:`tuple` repetition could fail to +respect :pep:`683`. + +.. + +.. date: 2023-05-01-21-05-47 +.. gh-issue: 104078 +.. nonce: vRaBsU +.. section: Core and Builtins + +Improve the performance of :c:func:`PyObject_HasAttrString` + +.. + +.. date: 2023-05-01-14-48-29 +.. gh-issue: 104066 +.. nonce: pzoUZQ +.. section: Core and Builtins + +Improve the performance of :func:`hasattr` for module objects with a missing +attribute. + +.. + +.. date: 2023-05-01-14-10-38 +.. gh-issue: 104028 +.. nonce: dxfh13 +.. section: Core and Builtins + +Reduce object creation while calling callback function from gc. Patch by +Dong-hee Na. + +.. + +.. date: 2023-05-01-12-03-52 +.. gh-issue: 104018 +.. nonce: PFxGS4 +.. section: Core and Builtins + +Disallow the "z" format specifier in %-format of bytes objects. + +.. + +.. date: 2023-05-01-08-08-05 +.. gh-issue: 102213 +.. nonce: nfH-4C +.. section: Core and Builtins + +Fix performance loss when accessing an object's attributes with +``__getattr__`` defined. + +.. + +.. date: 2023-04-26-17-56-18 +.. gh-issue: 103895 +.. nonce: ESB6tn +.. section: Core and Builtins + +Improve handling of edge cases in showing ``Exception.__notes__``. Ensures +that the messages always end with a newline and that string/bytes are not +exploded over multiple lines. Patch by Carey Metcalfe. + +.. + +.. date: 2023-04-26-16-26-35 +.. gh-issue: 103907 +.. nonce: kiONZQ +.. section: Core and Builtins + +Don't modify the refcounts of known immortal objects (:const:`True`, +:const:`False`, and :const:`None`) in the main interpreter loop. + +.. + +.. date: 2023-04-26-15-14-23 +.. gh-issue: 103899 +.. nonce: 1pqKPF +.. section: Core and Builtins + +Provide a helpful hint in the :exc:`TypeError` message when accidentally +calling a :term:`module` object that has a callable attribute of the same +name (such as :func:`dis.dis` or :class:`datetime.datetime`). + +.. + +.. date: 2023-04-25-20-56-01 +.. gh-issue: 103845 +.. nonce: V7NYFn +.. section: Core and Builtins + +Remove both line and instruction instrumentation before adding new ones for +monitoring, to avoid newly added instrumentation being removed immediately. + +.. + +.. date: 2023-04-25-08-43-11 +.. gh-issue: 103763 +.. nonce: ZLBZk1 +.. section: Core and Builtins + +Implement :pep:`695`, adding syntactic support for generic classes, generic +functions, and type aliases. + +A new ``type X = ...`` syntax is added for type aliases, which resolves at +runtime to an instance of the new class ``typing.TypeAliasType``. The value +is lazily evaluated and is accessible through the ``.__value__`` attribute. +This is implemented as a new AST node ``ast.TypeAlias``. + +New syntax (``class X[T]: ...``, ``def func[T](): ...``) is added for +defining generic functions and classes. This is implemented as a new +``type_params`` attribute on the AST nodes for classes and functions. This +node holds instances of the new AST classes ``ast.TypeVar``, +``ast.ParamSpec``, and ``ast.TypeVarTuple``. + +``typing.TypeVar``, ``typing.ParamSpec``, ``typing.ParamSpecArgs``, +``typing.ParamSpecKwargs``, ``typing.TypeVarTuple``, and ``typing.Generic`` +are now implemented in C rather than Python. + +There are new bytecode instructions ``LOAD_LOCALS``, +``LOAD_CLASSDICT_OR_GLOBAL``, and ``LOAD_CLASSDICT_OR_DEREF`` to support +correct resolution of names in class namespaces. + +Patch by Eric Traut, Larry Hastings, and Jelle Zijlstra. + +.. + +.. date: 2023-04-24-21-47-38 +.. gh-issue: 103801 +.. nonce: WaBanq +.. section: Core and Builtins + +Adds three minor linting fixes to the wasm module caught that were caught by +ruff. + +.. + +.. date: 2023-04-24-14-38-16 +.. gh-issue: 103793 +.. nonce: kqoH6Q +.. section: Core and Builtins + +Optimized asyncio Task creation by deferring expensive string formatting +(task name generation) from Task creation to the first time ``get_name`` is +called. This makes asyncio benchmarks up to 5% faster. + +.. + +.. date: 2023-04-21-17-03-14 +.. gh-issue: 102310 +.. nonce: anLjDx +.. section: Core and Builtins + +Change the error range for invalid bytes literals. + +.. + +.. date: 2023-04-21-16-12-41 +.. gh-issue: 103590 +.. nonce: 7DHDOE +.. section: Core and Builtins + +Do not wrap a single exception raised from a ``try-except*`` construct in an +:exc:`ExceptionGroup`. + +.. + +.. date: 2023-04-20-16-17-51 +.. gh-issue: 103650 +.. nonce: K1MFXR +.. section: Core and Builtins + +Change the perf map format to remove the '0x' prefix from the addresses + +.. + +.. date: 2023-04-17-16-00-32 +.. gh-issue: 102856 +.. nonce: UunJ7y +.. section: Core and Builtins + +Implement the required C tokenizer changes for PEP 701. Patch by Pablo +Galindo Salgado, Lysandros Nikolaou, Batuhan Taskaya, Marta Gómez Macías and +sunmy2019. + +.. + +.. date: 2023-04-16-14-38-39 +.. gh-issue: 100530 +.. nonce: OR6-sn +.. section: Core and Builtins + +Clarify the error message raised when the called part of a class pattern +isn't actually a class. + +.. + +.. date: 2023-04-14-22-35-23 +.. gh-issue: 101517 +.. nonce: 5EqM-S +.. section: Core and Builtins + +Fix bug in line numbers of instructions emitted for :keyword:`except* +`. + +.. + +.. date: 2023-04-13-00-58-55 +.. gh-issue: 103492 +.. nonce: P4k0Ay +.. section: Core and Builtins + +Clarify :exc:`SyntaxWarning` with literal ``is`` comparison by specifying +which literal is problematic, since comparisons using ``is`` with e.g. None +and bool literals are idiomatic. + +.. + +.. date: 2023-04-12-20-22-03 +.. gh-issue: 87729 +.. nonce: 99A7UO +.. section: Core and Builtins + +Add :opcode:`LOAD_SUPER_ATTR` (and a specialization for +``super().method()``) to speed up ``super().method()`` and ``super().attr``. +This makes ``super().method()`` roughly 2.3x faster and brings it within 20% +of the performance of a simple method call. Patch by Vladimir Matveev and +Carl Meyer. + +.. + +.. date: 2023-04-12-20-18-51 +.. gh-issue: 103488 +.. nonce: vYvlHD +.. section: Core and Builtins + +Change the internal offset distinguishing yield and return target addresses, +so that the instruction pointer is correct for exception handling and other +stack unwinding. + +.. + +.. date: 2023-04-12-19-55-24 +.. gh-issue: 82012 +.. nonce: FlcJAh +.. section: Core and Builtins + +The bitwise inversion operator (``~``) on bool is deprecated. It returns the +bitwise inversion of the underlying ``int`` representation such that +``bool(~True) == True``, which can be confusing. Use ``not`` for logical +negation of bools. In the rare case that you really need the bitwise +inversion of the underlying ``int``, convert to int explicitly ``~int(x)``. + +.. + +.. date: 2023-04-09-22-21-57 +.. gh-issue: 77757 +.. nonce: _Ow-u2 +.. section: Core and Builtins + +Exceptions raised in a typeobject's ``__set_name__`` method are no longer +wrapped by a :exc:`RuntimeError`. Context information is added to the +exception as a :pep:`678` note. + +.. + +.. date: 2023-04-09-04-30-02 +.. gh-issue: 103333 +.. nonce: gKOetS +.. section: Core and Builtins + +:exc:`AttributeError` now retains the ``name`` attribute when pickled and +unpickled. + +.. + +.. date: 2023-04-08-17-13-07 +.. gh-issue: 103242 +.. nonce: ysI1b3 +.. section: Core and Builtins + +Migrate :meth:`~ssl.SSLContext.set_ecdh_curve` method not to use deprecated +OpenSSL APIs. Patch by Dong-hee Na. + +.. + +.. date: 2023-04-07-12-18-41 +.. gh-issue: 103323 +.. nonce: 9802br +.. section: Core and Builtins + +We've replaced our use of ``_PyRuntime.tstate_current`` with a thread-local +variable. This is a fairly low-level implementation detail, and there +should be no change in behavior. + +.. + +.. date: 2023-04-02-22-14-57 +.. gh-issue: 84436 +.. nonce: hvMgwF +.. section: Core and Builtins + +The implementation of PEP-683 which adds Immortal Objects by using a fixed +reference count that skips reference counting to make objects truly +immutable. + +.. + +.. date: 2023-04-01-00-46-31 +.. gh-issue: 102700 +.. nonce: 493NB4 +.. section: Core and Builtins + +Allow built-in modules to be submodules. This allows submodules to be +statically linked into a CPython binary. + +.. + +.. date: 2023-03-31-17-24-03 +.. gh-issue: 103082 +.. nonce: isRUcV +.. section: Core and Builtins + +Implement :pep:`669` Low Impact Monitoring for CPython. + +.. + +.. date: 2023-03-25-23-24-38 +.. gh-issue: 88691 +.. nonce: 2SWBd1 +.. section: Core and Builtins + +Reduce the number of inline :opcode:`CACHE` entries for :opcode:`CALL`. + +.. + +.. date: 2023-03-07-17-37-00 +.. gh-issue: 102500 +.. nonce: RUSQhz +.. section: Core and Builtins + +Make the buffer protocol accessible in Python code using the new +``__buffer__`` and ``__release_buffer__`` magic methods. See :pep:`688` for +details. Patch by Jelle Zijlstra. + +.. + +.. date: 2023-01-30-15-40-29 +.. gh-issue: 97933 +.. nonce: nUlp3r +.. section: Core and Builtins + +:pep:`709`: inline list, dict and set comprehensions to improve performance +and reduce bytecode size. + +.. + +.. date: 2022-11-08-12-36-25 +.. gh-issue: 99184 +.. nonce: KIaqzz +.. section: Core and Builtins + +Bypass instance attribute access of ``__name__`` in ``repr`` of +:class:`weakref.ref`. + +.. + +.. date: 2022-10-06-23-32-11 +.. gh-issue: 98003 +.. nonce: xWE0Yu +.. section: Core and Builtins + +Complex function calls are now faster and consume no C stack space. + +.. + +.. bpo: 39610 +.. date: 2020-02-11-15-54-40 +.. nonce: fvgsCl +.. section: Core and Builtins + +``len()`` for 0-dimensional :class:`memoryview`` objects (such as +``memoryview(ctypes.c_uint8(42))``) now raises a :exc:`TypeError`. +Previously this returned ``1``, which was not consistent with ``mem_0d[0]`` +raising an :exc:`IndexError``. + +.. + +.. bpo: 31821 +.. date: 2019-12-01-12-58-31 +.. nonce: 1FNmwk +.. section: Core and Builtins + +Fix :func:`!pause_reading` to work when called from :func:`!connection_made` +in :mod:`asyncio`. + +.. + +.. date: 2023-05-17-21-01-48 +.. gh-issue: 104600 +.. nonce: E6CK35 +.. section: Library + +:func:`functools.update_wrapper` now sets the ``__type_params__`` attribute +(added by :pep:`695`). + +.. + +.. date: 2023-05-17-20-03-01 +.. gh-issue: 104340 +.. nonce: kp_XmX +.. section: Library + +When an ``asyncio`` pipe protocol loses its connection due to an error, and +the caller doesn't await ``wait_closed()`` on the corresponding +``StreamWriter``, don't log a warning about an exception that was never +retrieved. After all, according to the ``StreamWriter.close()`` docs, the +``wait_closed()`` call is optional ("not mandatory"). + +.. + +.. date: 2023-05-17-16-58-23 +.. gh-issue: 104555 +.. nonce: 5rb5oM +.. section: Library + +Fix issue where an :func:`issubclass` check comparing a class ``X`` against +a :func:`runtime-checkable protocol ` ``Y`` with +non-callable members would not cause :exc:`TypeError` to be raised if an +:func:`isinstance` call had previously been made comparing an instance of +``X`` to ``Y``. This issue was present in edge cases on Python 3.11, but +became more prominent in 3.12 due to some unrelated changes that were made +to runtime-checkable protocols. Patch by Alex Waygood. + +.. + +.. date: 2023-05-17-08-01-36 +.. gh-issue: 104372 +.. nonce: jpoWs6 +.. section: Library + +Refactored the ``_posixsubprocess`` internals to avoid Python C API usage +between fork and exec when marking ``pass_fds=`` file descriptors +inheritable. + +.. + +.. date: 2023-05-17-03-14-07 +.. gh-issue: 104484 +.. nonce: y6KxL6 +.. section: Library + +Added *case_sensitive* argument to :meth:`pathlib.PurePath.match` + +.. + +.. date: 2023-05-16-11-02-44 +.. gh-issue: 75367 +.. nonce: qLWR35 +.. section: Library + +Fix data descriptor detection in :func:`inspect.getattr_static`. + +.. + +.. date: 2023-05-16-10-07-16 +.. gh-issue: 104536 +.. nonce: hFWD8f +.. section: Library + +Fix a race condition in the internal :mod:`multiprocessing.process` cleanup +logic that could manifest as an unintended ``AttributeError`` when calling +``process.close()``. + +.. + +.. date: 2023-05-12-19-29-28 +.. gh-issue: 103857 +.. nonce: 0IzSxr +.. section: Library + +Update datetime deprecations' stracktrace to point to the calling line + +.. + +.. date: 2023-05-11-21-32-18 +.. gh-issue: 101520 +.. nonce: l9MjRE +.. section: Library + +Move the core functionality of the ``tracemalloc`` module in the ``Python/`` +folder, leaving just the module wrapper in ``Modules/``. + +.. + +.. date: 2023-05-11-07-50-00 +.. gh-issue: 104392 +.. nonce: YSllzt +.. section: Library + +Remove undocumented and unused ``_paramspec_tvars`` attribute from some +classes in :mod:`typing`. + +.. + +.. date: 2023-05-11-01-07-42 +.. gh-issue: 102613 +.. nonce: uMsokt +.. section: Library + +Fix issue where :meth:`pathlib.Path.glob` raised :exc:`RecursionError` when +walking deep directory trees. + +.. + +.. date: 2023-05-10-19-33-36 +.. gh-issue: 103000 +.. nonce: j0KSfD +.. section: Library + +Improve performance of :func:`dataclasses.asdict` for the common case where +*dict_factory* is ``dict``. Patch by David C Ellis. + +.. + +.. date: 2023-05-09-18-46-24 +.. gh-issue: 104301 +.. nonce: gNnbId +.. section: Library + +Allow leading whitespace in disambiguated statements in :mod:`pdb`. + +.. + +.. date: 2023-05-08-23-01-59 +.. gh-issue: 104139 +.. nonce: 83Tnt- +.. section: Library + +Teach :func:`urllib.parse.unsplit` to retain the ``"//"`` when assembling +``itms-services://?action=generate-bugs`` style `Apple Platform Deployment +`_ URLs. + +.. + +.. date: 2023-05-08-20-57-17 +.. gh-issue: 104307 +.. nonce: DSB93G +.. section: Library + +:func:`socket.getnameinfo` now releases the GIL while contacting the DNS +server + +.. + +.. date: 2023-05-08-15-50-59 +.. gh-issue: 104310 +.. nonce: fXVSPY +.. section: Library + +Users may now use ``importlib.util.allowing_all_extensions()`` (a context +manager) to temporarily disable the strict compatibility checks for +importing extension modules in subinterpreters. + +.. + +.. date: 2023-05-08-15-39-00 +.. gh-issue: 87695 +.. nonce: f6iO7v +.. section: Library + +Fix issue where :meth:`pathlib.Path.glob` raised :exc:`OSError` when it +encountered a symlink to an overly long path. + +.. + +.. date: 2023-05-07-19-56-45 +.. gh-issue: 104265 +.. nonce: fVblry +.. section: Library + +Prevent possible crash by disallowing instantiation of the +:class:`!_csv.Reader` and :class:`!_csv.Writer` types. The regression was +introduced in 3.10.0a4 with PR 23224 (:issue:`14935`). Patch by Radislav +Chugunov. + +.. + +.. date: 2023-05-06-20-37-46 +.. gh-issue: 102613 +.. nonce: QZG9iX +.. section: Library + +Improve performance of :meth:`pathlib.Path.glob` when expanding recursive +wildcards ("``**``") by merging adjacent wildcards and de-duplicating +results only when necessary. + +.. + +.. date: 2023-05-05-18-52-22 +.. gh-issue: 65772 +.. nonce: w5P5Wv +.. section: Library + +Remove unneeded comments and code in turtle.py. + +.. + +.. date: 2023-05-03-19-22-24 +.. gh-issue: 90208 +.. nonce: tI00da +.. section: Library + +Fixed issue where :meth:`pathlib.Path.glob` returned incomplete results when +it encountered a :exc:`PermissionError`. This method now suppresses all +:exc:`OSError` exceptions, except those raised from calling +:meth:`~pathlib.Path.is_dir` on the top-level path. + +.. + +.. date: 2023-05-03-16-51-53 +.. gh-issue: 104144 +.. nonce: 653Q0P +.. section: Library + +Optimize :class:`asyncio.TaskGroup` when using +:func:`asyncio.eager_task_factory`. Skip scheduling a done callback if a +TaskGroup task completes eagerly. + +.. + +.. date: 2023-05-03-16-50-24 +.. gh-issue: 104144 +.. nonce: yNkjL8 +.. section: Library + +Optimize :func:`asyncio.gather` when using +:func:`asyncio.eager_task_factory` to complete eagerly if all fututres +completed eagerly. Avoid scheduling done callbacks for futures that complete +eagerly. + +.. + +.. date: 2023-05-03-03-14-33 +.. gh-issue: 104114 +.. nonce: RG26RD +.. section: Library + +Fix issue where :meth:`pathlib.Path.glob` returns paths using the case of +non-wildcard segments for corresponding path segments, rather than the real +filesystem case. + +.. + +.. date: 2023-05-02-21-05-30 +.. gh-issue: 104104 +.. nonce: 9tjplT +.. section: Library + +Improve performance of :meth:`pathlib.Path.glob` by using +:data:`re.IGNORECASE` to implement case-insensitive matching. + +.. + +.. date: 2023-05-02-20-43-03 +.. gh-issue: 104102 +.. nonce: vgSdEJ +.. section: Library + +Improve performance of :meth:`pathlib.Path.glob` when evaluating patterns +that contain ``'../'`` segments. + +.. + +.. date: 2023-05-02-04-49-45 +.. gh-issue: 103822 +.. nonce: m0QdAO +.. section: Library + +Update the return type of ``weekday`` to the newly added Day attribute + +.. + +.. date: 2023-05-01-19-10-05 +.. gh-issue: 103629 +.. nonce: 81bpZz +.. section: Library + +Update the ``repr`` of :class:`typing.Unpack` according to :pep:`692`. + +.. + +.. date: 2023-05-01-17-58-28 +.. gh-issue: 103963 +.. nonce: XWlHx7 +.. section: Library + +Make :mod:`dis` display the names of the args for +:opcode:`CALL_INTRINSIC_*`. + +.. + +.. date: 2023-05-01-16-43-28 +.. gh-issue: 104035 +.. nonce: MrJBw8 +.. section: Library + +Do not ignore user-defined ``__getstate__`` and ``__setstate__`` methods for +slotted frozen dataclasses. + +.. + +.. date: 2023-04-29-18-23-16 +.. gh-issue: 103987 +.. nonce: sRgALL +.. section: Library + +In :mod:`mmap`, fix several bugs that could lead to access to memory-mapped +files after they have been invalidated. + +.. + +.. date: 2023-04-28-19-08-50 +.. gh-issue: 103977 +.. nonce: msF70A +.. section: Library + +Improve import time of :mod:`platform` module. + +.. + +.. date: 2023-04-28-18-04-23 +.. gh-issue: 88773 +.. nonce: xXCNJw +.. section: Library + +Added :func:`turtle.teleport` to the :mod:`turtle` module to move a turtle +to a new point without tracing a line, visible or invisible. Patch by Liam +Gersten. + +.. + +.. date: 2023-04-27-20-03-08 +.. gh-issue: 103935 +.. nonce: Uaf2M0 +.. section: Library + +Use :func:`io.open_code` for files to be executed instead of raw +:func:`open` + +.. + +.. date: 2023-04-27-18-46-31 +.. gh-issue: 68968 +.. nonce: E3tnhy +.. section: Library + +Fixed garbled output of :meth:`~unittest.TestCase.assertEqual` when an input +lacks final newline. + +.. + +.. date: 2023-04-27-00-45-41 +.. gh-issue: 100370 +.. nonce: MgZ3KY +.. section: Library + +Fix potential :exc:`OverflowError` in :meth:`sqlite3.Connection.blobopen` +for 32-bit builds. Patch by Erlend E. Aasland. + +.. + +.. date: 2023-04-27-00-05-32 +.. gh-issue: 102628 +.. nonce: X230E- +.. section: Library + +Substitute CTRL-D with CTRL-Z in :mod:`sqlite3` CLI banner when running on +Windows. + +.. + +.. date: 2023-04-26-18-12-13 +.. gh-issue: 103636 +.. nonce: -KvCgO +.. section: Library + +Module-level attributes ``January`` and ``February`` are deprecated from +:mod:`calendar`. + +.. + +.. date: 2023-04-26-15-14-36 +.. gh-issue: 103583 +.. nonce: iCMDFt +.. section: Library + +Isolate :mod:`!_multibytecodec` and codecs extension modules. Patches by +Erlend E. Aasland. + +.. + +.. date: 2023-04-26-09-54-25 +.. gh-issue: 103848 +.. nonce: aDSnpR +.. section: Library + +Add checks to ensure that ``[`` bracketed ``]`` hosts found by +:func:`urllib.parse.urlsplit` are of IPv6 or IPvFuture format. + +.. + +.. date: 2023-04-26-09-38-47 +.. gh-issue: 103872 +.. nonce: 8LBsDz +.. section: Library + +Update the bundled copy of pip to version 23.1.2. + +.. + +.. date: 2023-04-25-22-59-06 +.. gh-issue: 99944 +.. nonce: pst8iT +.. section: Library + +Make :mod:`dis` display the value of oparg of :opcode:`KW_NAMES`. + +.. + +.. date: 2023-04-25-22-06-00 +.. gh-issue: 74940 +.. nonce: TOacQ9 +.. section: Library + +The C.UTF-8 locale is no longer converted to en_US.UTF-8, enabling the use +of UTF-8 encoding on systems which have no locales installed. + +.. + +.. date: 2023-04-25-19-58-13 +.. gh-issue: 103861 +.. nonce: JeozgD +.. section: Library + +Fix ``zipfile.Zipfile`` creating invalid zip files when ``force_zip64`` was +used to add files to them. Patch by Carey Metcalfe. + +.. + +.. date: 2023-04-25-17-03-18 +.. gh-issue: 103857 +.. nonce: Mr2Cak +.. section: Library + +Deprecated :meth:`datetime.datetime.utcnow` and +:meth:`datetime.datetime.utcfromtimestamp`. (Patch by Paul Ganssle) + +.. + +.. date: 2023-04-25-16-31-00 +.. gh-issue: 103839 +.. nonce: tpyLhI +.. section: Library + +Avoid compilation error due to tommath.h not being found when building +Tkinter against Tcl 8.7 built with bundled libtommath. + +.. + +.. date: 2023-04-24-23-07-56 +.. gh-issue: 103791 +.. nonce: bBPWdS +.. section: Library + +:class:`contextlib.suppress` now supports suppressing exceptions raised as +part of an :exc:`ExceptionGroup`. If other exceptions exist on the group, +they are re-raised in a group that does not contain the suppressed +exceptions. + +.. + +.. date: 2023-04-24-16-00-28 +.. gh-issue: 90750 +.. nonce: da0Xi8 +.. section: Library + +Use :meth:`datetime.datetime.fromisocalendar` in the implementation of +:meth:`datetime.datetime.strptime`, which should now accept only valid ISO +dates. (Patch by Paul Ganssle) + +.. + +.. date: 2023-04-24-00-34-23 +.. gh-issue: 103685 +.. nonce: U14jBM +.. section: Library + +Prepare :meth:`tkinter.Menu.index` for Tk 8.7 so that it does not raise +``TclError: expected integer but got ""`` when it should return ``None``. + +.. + +.. date: 2023-04-23-15-39-17 +.. gh-issue: 81403 +.. nonce: zVz9Td +.. section: Library + +:class:`urllib.request.CacheFTPHandler` no longer raises :class:`URLError` +if a cached FTP instance is reused. ftplib's endtransfer method calls +voidresp to drain the connection to handle FTP instance reuse properly. + +.. + +.. date: 2023-04-22-22-37-39 +.. gh-issue: 103699 +.. nonce: NizCjc +.. section: Library + +Add ``__orig_bases__`` to non-generic TypedDicts, call-based TypedDicts, and +call-based NamedTuples. Other TypedDicts and NamedTuples already had the +attribute. + +.. + +.. date: 2023-04-22-21-34-13 +.. gh-issue: 103693 +.. nonce: SBtuLQ +.. section: Library + +Add convenience variable feature to :mod:`pdb` + +.. + +.. date: 2023-04-22-12-30-10 +.. gh-issue: 92248 +.. nonce: NcVTKR +.. section: Library + +Deprecate ``type``, ``choices``, and ``metavar`` parameters of +``argparse.BooleanOptionalAction``. + +.. + +.. date: 2023-04-22-11-20-27 +.. gh-issue: 89415 +.. nonce: YHk760 +.. section: Library + +Add :mod:`socket` constants for source-specific multicast. Patch by Reese +Hyde. + +.. + +.. date: 2023-04-22-02-41-06 +.. gh-issue: 103673 +.. nonce: oE7S_k +.. section: Library + +:mod:`socketserver` gains ``ForkingUnixStreamServer`` and +``ForkingUnixDatagramServer`` classes. Patch by Jay Berry. + +.. + +.. date: 2023-04-21-10-25-39 +.. gh-issue: 103636 +.. nonce: YK6NEa +.. section: Library + +Added Enum for months and days in the calendar module. + +.. + +.. date: 2023-04-19-16-08-53 +.. gh-issue: 84976 +.. nonce: HwbzlD +.. section: Library + +Create a new ``Lib/_pydatetime.py`` file that defines the Python version of +the ``datetime`` module, and make ``datetime`` import the contents of the +new library only if the C implementation is missing. Currently, the full +Python implementation is defined and then deleted if the C implementation is +not available, slowing down ``import datetime`` unnecessarily. + +.. + +.. date: 2023-04-17-14-47-28 +.. gh-issue: 103596 +.. nonce: ME1y3_ +.. section: Library + +Attributes/methods are no longer shadowed by same-named enum members, +although they may be shadowed by enum.property's. + +.. + +.. date: 2023-04-16-19-48-21 +.. gh-issue: 103584 +.. nonce: 3mBTuM +.. section: Library + +Updated ``importlib.metadata`` with changes from ``importlib_metadata`` 5.2 +through 6.5.0, including: Support ``installed-files.txt`` for +``Distribution.files`` when present. ``PackageMetadata`` now stipulates an +additional ``get`` method allowing for easy querying of metadata keys that +may not be present. ``packages_distributions`` now honors packages and +modules with Python modules that not ``.py`` sources (e.g. ``.pyc``, +``.so``). Expand protocol for ``PackageMetadata.get_all`` to match the +upstream implementation of ``email.message.Message.get_all`` in +python/typeshed#9620. Deprecated use of ``Distribution`` without defining +abstract methods. Deprecated expectation that +``PackageMetadata.__getitem__`` will return ``None`` for missing keys. In +the future, it will raise a ``KeyError``. + +.. + +.. date: 2023-04-16-18-29-04 +.. gh-issue: 103578 +.. nonce: fly1wc +.. section: Library + +Fixed a bug where :mod:`pdb` crashes when reading source file with different +encoding by replacing :func:`io.open` with :func:`io.open_code`. The new +method would also call into the hook set by :func:`PyFile_SetOpenCodeHook`. + +.. + +.. date: 2023-04-15-12-19-14 +.. gh-issue: 103556 +.. nonce: TEf-2m +.. section: Library + +Now creating :class:`inspect.Signature` objects with positional-only +parameter with a default followed by a positional-or-keyword parameter +without one is impossible. + +.. + +.. date: 2023-04-15-11-21-38 +.. gh-issue: 103559 +.. nonce: a9rYHG +.. section: Library + +Update the bundled copy of pip to version 23.1.1. + +.. + +.. date: 2023-04-14-21-16-05 +.. gh-issue: 103548 +.. nonce: lagdpp +.. section: Library + +Improve performance of :meth:`pathlib.Path.absolute` and +:meth:`~pathlib.Path.cwd` by joining paths only when necessary. Also improve +performance of :meth:`pathlib.PurePath.is_absolute` on Posix by skipping +path parsing and normalization. + +.. + +.. date: 2023-04-14-21-12-32 +.. gh-issue: 103538 +.. nonce: M4FK_v +.. section: Library + +Remove ``_tkinter`` module code guarded by definition of the ``TK_AQUA`` +macro which was only needed for Tk 8.4.7 or earlier and was never actually +defined by any build system or documented for manual use. + +.. + +.. date: 2023-04-14-06-32-54 +.. gh-issue: 103533 +.. nonce: n_AfcS +.. section: Library + +Update :mod:`cProfile` to use PEP 669 API + +.. + +.. date: 2023-04-13-19-43-15 +.. gh-issue: 103525 +.. nonce: uY4VYg +.. section: Library + +Fix misleading exception message when mixed ``str`` and ``bytes`` arguments +are supplied to :class:`pathlib.PurePath` and :class:`~pathlib.Path`. + +.. + +.. date: 2023-04-13-13-17-47 +.. gh-issue: 103489 +.. nonce: ZSZgmu +.. section: Library + +Add :meth:`~sqlite3.Connection.getconfig` and +:meth:`~sqlite3.Connection.setconfig` to :class:`~sqlite3.Connection` to +make configuration changes to a database connection. Patch by Erlend E. +Aasland. + +.. + +.. date: 2023-04-12-17-59-55 +.. gh-issue: 103365 +.. nonce: UBEE0U +.. section: Library + +Set default Flag boundary to ``STRICT`` and fix bitwise operations. + +.. + +.. date: 2023-04-12-13-04-16 +.. gh-issue: 103472 +.. nonce: C6bOHv +.. section: Library + +Avoid a potential :exc:`ResourceWarning` in +:class:`http.client.HTTPConnection` by closing the proxy / tunnel's CONNECT +response explicitly. + +.. + +.. date: 2023-04-12-06-00-02 +.. gh-issue: 103462 +.. nonce: w6yBlM +.. section: Library + +Fixed an issue with using :meth:`~asyncio.WriteTransport.writelines` in +:mod:`asyncio` to send very large payloads that exceed the amount of data +that can be written in one call to :meth:`socket.socket.send` or +:meth:`socket.socket.sendmsg`, resulting in the remaining buffer being left +unwritten. + +.. + +.. date: 2023-04-11-21-38-39 +.. gh-issue: 103449 +.. nonce: -nxmhb +.. section: Library + +Fix a bug in doc string generation in :func:`dataclasses.dataclass`. + +.. + +.. date: 2023-04-09-06-59-36 +.. gh-issue: 103092 +.. nonce: vskbro +.. section: Library + +Isolate :mod:`!_collections` (apply :pep:`687`). Patch by Erlend E. Aasland. + +.. + +.. date: 2023-04-08-01-33-12 +.. gh-issue: 103357 +.. nonce: vjin28 +.. section: Library + +Added support for :class:`logging.Formatter` ``defaults`` parameter to +:func:`logging.config.dictConfig` and :func:`logging.config.fileConfig`. +Patch by Bar Harel. + +.. + +.. date: 2023-04-08-00-48-40 +.. gh-issue: 103092 +.. nonce: 5EFts0 +.. section: Library + +Adapt the :mod:`winreg` extension module to :pep:`687`. + +.. + +.. date: 2023-04-07-15-15-40 +.. gh-issue: 74690 +.. nonce: un84hh +.. section: Library + +The performance of :func:`isinstance` checks against +:func:`runtime-checkable protocols ` has been +considerably improved for protocols that only have a few members. To achieve +this improvement, several internal implementation details of the +:mod:`typing` module have been refactored, including +``typing._ProtocolMeta.__instancecheck__``, +``typing._is_callable_members_only``, and ``typing._get_protocol_attrs``. +Patches by Alex Waygood. + +.. + +.. date: 2023-04-07-15-09-26 +.. gh-issue: 74690 +.. nonce: 0f886b +.. section: Library + +The members of a runtime-checkable protocol are now considered "frozen" at +runtime as soon as the class has been created. See :ref:`"What's new in +Python 3.12" ` for more details. + +.. + +.. date: 2023-04-06-17-28-36 +.. gh-issue: 103256 +.. nonce: 1syxfs +.. section: Library + +Fixed a bug that caused :mod:`hmac` to raise an exception when the requested +hash algorithm was not available in OpenSSL despite being available +separately as part of ``hashlib`` itself. It now falls back properly to the +built-in. This could happen when, for example, your OpenSSL does not include +SHA3 support and you want to compute ``hmac.digest(b'K', b'M', +'sha3_256')``. + +.. + +.. date: 2023-04-06-16-55-51 +.. gh-issue: 102778 +.. nonce: BWeAmE +.. section: Library + +Support ``sys.last_exc`` in :mod:`idlelib`. + +.. + +.. date: 2023-04-06-04-35-59 +.. gh-issue: 103285 +.. nonce: rCZ9-G +.. section: Library + +Improve performance of :func:`ast.get_source_segment`. + +.. + +.. date: 2023-04-05-01-28-53 +.. gh-issue: 103225 +.. nonce: QD3JVU +.. section: Library + +Fix a bug in :mod:`pdb` when displaying line numbers of module-level source +code. + +.. + +.. date: 2023-04-04-21-44-25 +.. gh-issue: 103092 +.. nonce: Dz0_Xn +.. section: Library + +Adapt the :mod:`msvcrt` extension module to :pep:`687`. + +.. + +.. date: 2023-04-04-21-27-51 +.. gh-issue: 103092 +.. nonce: 7s7Bzf +.. section: Library + +Adapt the :mod:`winsound` extension module to :pep:`687`. + +.. + +.. date: 2023-04-04-12-43-38 +.. gh-issue: 93910 +.. nonce: jurMzv +.. section: Library + +Remove deprecation of enum ``memmber.member`` access. + +.. + +.. date: 2023-04-03-23-44-34 +.. gh-issue: 102978 +.. nonce: gy9eVk +.. section: Library + +Fixes :func:`unittest.mock.patch` not enforcing function signatures for +methods decorated with ``@classmethod`` or ``@staticmethod`` when patch is +called with ``autospec=True``. + +.. + +.. date: 2023-04-03-23-43-12 +.. gh-issue: 103092 +.. nonce: 3xqk4y +.. section: Library + +Isolate :mod:`!_socket` (apply :pep:`687`). Patch by Erlend E. Aasland. + +.. + +.. date: 2023-04-03-22-02-35 +.. gh-issue: 100479 +.. nonce: kNBjQm +.. section: Library + +Add :meth:`pathlib.PurePath.with_segments`, which creates a path object from +arguments. This method is called whenever a derivative path is created, such +as from :attr:`pathlib.PurePath.parent`. Subclasses may override this method +to share information between path objects. + +.. + +.. date: 2023-04-03-21-08-53 +.. gh-issue: 103220 +.. nonce: OW_Bj5 +.. section: Library + +Fix issue where :func:`os.path.join` added a slash when joining onto an +incomplete UNC drive with a trailing slash on Windows. + +.. + +.. date: 2023-04-02-23-05-22 +.. gh-issue: 103204 +.. nonce: bbDmu0 +.. section: Library + +Fixes :mod:`http.server` accepting HTTP requests with HTTP version numbers +preceded by '+', or '-', or with digit-separating '_' characters. The +length of the version numbers is also constrained. + +.. + +.. date: 2023-04-02-22-04-26 +.. gh-issue: 75586 +.. nonce: 526iJm +.. section: Library + +Fix various Windows-specific issues with ``shutil.which``. + +.. + +.. date: 2023-04-02-17-51-08 +.. gh-issue: 103193 +.. nonce: xrZbM1 +.. section: Library + +Improve performance of :func:`inspect.getattr_static`. Patch by Alex +Waygood. + +.. + +.. date: 2023-04-01-23-01-31 +.. gh-issue: 103176 +.. nonce: FBsdxa +.. section: Library + +:func:`sys._current_exceptions` now returns a mapping from thread-id to an +exception instance, rather than to a ``(typ, exc, tb)`` tuple. + +.. + +.. date: 2023-03-31-01-13-00 +.. gh-issue: 103143 +.. nonce: 6eMluy +.. section: Library + +Polish the help messages and docstrings of :mod:`pdb`. + +.. + +.. date: 2023-03-28-09-13-31 +.. gh-issue: 103015 +.. nonce: ETTfNf +.. section: Library + +Add *entrypoint* keyword-only parameter to +:meth:`sqlite3.Connection.load_extension`, for overriding the SQLite +extension entry point. Patch by Erlend E. Aasland. + +.. + +.. date: 2023-03-24-20-49-48 +.. gh-issue: 103000 +.. nonce: 6eVNZI +.. section: Library + +Improve performance of :func:`dataclasses.astuple` and +:func:`dataclasses.asdict` in cases where the contents are common Python +types. + +.. + +.. date: 2023-03-23-15-24-38 +.. gh-issue: 102953 +.. nonce: YR4KaK +.. section: Library + +The extraction methods in :mod:`tarfile`, and :func:`shutil.unpack_archive`, +have a new a *filter* argument that allows limiting tar features than may be +surprising or dangerous, such as creating files outside the destination +directory. See :ref:`tarfile-extraction-filter` for details. + +.. + +.. date: 2023-03-15-12-18-07 +.. gh-issue: 97696 +.. nonce: DtnpIC +.. section: Library + +Implemented an eager task factory in asyncio. When used as a task factory on +an event loop, it performs eager execution of coroutines. Coroutines that +are able to complete synchronously (e.g. return or raise without blocking) +are returned immediately as a finished task, and the task is never scheduled +to the event loop. If the coroutine blocks, the (pending) task is scheduled +and returned. + +.. + +.. date: 2023-03-15-00-37-43 +.. gh-issue: 81079 +.. nonce: heTAod +.. section: Library + +Add *case_sensitive* keyword-only argument to :meth:`pathlib.Path.glob` and +:meth:`~pathlib.Path.rglob`. + +.. + +.. date: 2023-03-14-11-20-19 +.. gh-issue: 101819 +.. nonce: 0-h0it +.. section: Library + +Isolate the :mod:`io` extension module by applying :pep:`687`. Patch by +Kumar Aditya, Victor Stinner, and Erlend E. Aasland. + +.. + +.. date: 2023-03-08-02-45-46 +.. gh-issue: 91896 +.. nonce: kgON_a +.. section: Library + +Deprecate :class:`collections.abc.ByteString` + +.. + +.. date: 2023-03-06-18-49-57 +.. gh-issue: 101362 +.. nonce: eSSy6L +.. section: Library + +Speed up :class:`pathlib.Path` construction by omitting the path anchor from +the internal list of path parts. + +.. + +.. date: 2023-02-21-14-57-34 +.. gh-issue: 102114 +.. nonce: uUDQzb +.. section: Library + +Functions in the :mod:`dis` module that accept a source code string as +argument now print a more concise traceback when the string contains a +syntax or indentation error. + +.. + +.. date: 2023-02-19-12-37-08 +.. gh-issue: 62432 +.. nonce: GnBFIB +.. section: Library + +The :mod:`unittest` runner will now exit with status code 5 if no tests were +run. It is common for test runner misconfiguration to fail to find any +tests, this should be an error. + +.. + +.. date: 2023-02-17-21-14-40 +.. gh-issue: 78079 +.. nonce: z3Szr6 +.. section: Library + +Fix incorrect normalization of UNC device path roots, and partial UNC share +path roots, in :class:`pathlib.PurePath`. Pathlib no longer appends a +trailing slash to such paths. + +.. + +.. date: 2023-02-11-21-18-10 +.. gh-issue: 85984 +.. nonce: nvzOD0 +.. section: Library + +Add :func:`tty.cfmakeraw` and :func:`tty.cfmakecbreak` to :mod:`tty` and +modernize, the behavior of :func:`tty.setraw` and :func:`tty.setcbreak` to +use POSIX.1-2017 Chapter 11 "General Terminal Interface" flag masks by +default. + +.. + +.. date: 2023-02-11-15-01-32 +.. gh-issue: 101688 +.. nonce: kwXmfM +.. section: Library + +Implement :func:`types.get_original_bases` to provide further introspection +for types. + +.. + +.. date: 2023-02-09-22-24-34 +.. gh-issue: 101640 +.. nonce: oFuEpB +.. section: Library + +:class:`argparse.ArgumentParser` now catches errors when writing messages, +such as when :data:`sys.stderr` is ``None``. Patch by Oleg Iarygin. + +.. + +.. date: 2023-02-06-16-45-18 +.. gh-issue: 83861 +.. nonce: mMbIU3 +.. section: Library + +Fix datetime.astimezone method return value when invoked on a naive datetime +instance that represents local time falling in a timezone transition gap. +PEP 495 requires that instances with fold=1 produce earlier times than those +with fold=0 in this case. + +.. + +.. date: 2023-01-22-14-53-12 +.. gh-issue: 89550 +.. nonce: c1U23f +.. section: Library + +Decrease execution time of some :mod:`gzip` file writes by 15% by adding +more appropriate buffering. + +.. + +.. date: 2023-01-14-17-54-56 +.. gh-issue: 95299 +.. nonce: vUhpKz +.. section: Library + +Remove the bundled setuptools wheel from ``ensurepip``, and stop installing +setuptools in environments created by ``venv``. + +.. + +.. date: 2022-11-10-16-26-47 +.. gh-issue: 99353 +.. nonce: DQFjnt +.. section: Library + +Respect the :class:`http.client.HTTPConnection` ``.debuglevel`` flag in +:class:`urllib.request.AbstractHTTPHandler` when its constructor parameter +``debuglevel`` is not set. And do the same for ``*HTTPS*``. + +.. + +.. date: 2022-10-21-17-20-57 +.. gh-issue: 98040 +.. nonce: 3btbmA +.. section: Library + +Remove the long-deprecated ``imp`` module. + +.. + +.. date: 2022-10-21-16-23-31 +.. gh-issue: 97850 +.. nonce: N46coo +.. section: Library + +Deprecate :func:`pkgutil.find_loader` and :func:`pkgutil.get_loader` in +favor of :func:`importlib.util.find_spec`. + +.. + +.. date: 2022-10-20-14-03-58 +.. gh-issue: 94473 +.. nonce: pzGX73 +.. section: Library + +Flatten arguments in :meth:`tkinter.Canvas.coords`. It now accepts not only +``x1, y1, x2, y2, ...`` and ``[x1, y1, x2, y2, ...]``, but also ``(x1, y1), +(x2, y2), ...`` and ``[(x1, y1), (x2, y2), ...]``. + +.. + +.. date: 2022-10-09-14-47-42 +.. gh-issue: 98040 +.. nonce: IN3qab +.. section: Library + +Remove more deprecated importlib APIs: ``find_loader()``, ``find_module()``, +``importlib.abc.Finder``, ``pkgutil.ImpImporter``, ``pkgutil.ImpLoader``. + +.. + +.. date: 2022-09-07-09-32-07 +.. gh-issue: 96522 +.. nonce: t73oqp +.. section: Library + +Fix potential deadlock in pty.spawn() + +.. + +.. date: 2022-09-03-09-24-02 +.. gh-issue: 96534 +.. nonce: EU4Oxv +.. section: Library + +Support divert(4) added in FreeBSD 14. + +.. + +.. date: 2022-08-27-21-41-41 +.. gh-issue: 87474 +.. nonce: 9X-kxt +.. section: Library + +Fix potential file descriptor leaks in :class:`subprocess.Popen`. + +.. + +.. date: 2022-07-16-17-15-29 +.. gh-issue: 94906 +.. nonce: C4G8DG +.. section: Library + +Support multiple steps in :func:`math.nextafter`. Patch by Shantanu Jain and +Matthias Gorgens. + +.. + +.. date: 2022-07-06-11-10-37 +.. gh-issue: 51574 +.. nonce: sveUeD +.. section: Library + +Make :func:`tempfile.mkdtemp` return absolute paths when its *dir* parameter +is relative. + +.. + +.. date: 2022-07-03-23-13-28 +.. gh-issue: 94518 +.. nonce: 511Tbh +.. section: Library + +Convert private :meth:`_posixsubprocess.fork_exec` to use Argument Clinic. + +.. + +.. date: 2022-05-02-16-21-05 +.. gh-issue: 92184 +.. nonce: hneGVW +.. section: Library + +When creating zip files using :mod:`zipfile`, ``os.altsep``, if not +``None``, will always be treated as a path separator even when it is not +``/``. Patch by Carey Metcalfe. + +.. + +.. bpo: 46797 +.. date: 2022-02-19-14-19-34 +.. nonce: 6BXZX4 +.. section: Library + +Deprecation warnings are now emitted for :class:`!ast.Num`, +:class:`!ast.Bytes`, :class:`!ast.Str`, :class:`!ast.NameConstant` and +:class:`!ast.Ellipsis`. These have been documented as deprecated since +Python 3.8, and will be removed in Python 3.14. + +.. + +.. bpo: 44844 +.. date: 2021-12-03-23-00-56 +.. nonce: tvg2VY +.. section: Library + +Enables :mod:`webbrowser` to detect and launch Microsoft Edge browser. + +.. + +.. bpo: 45606 +.. date: 2021-11-19-23-37-18 +.. nonce: UW5XE1 +.. section: Library + +Fixed the bug in :meth:`pathlib.Path.glob` -- previously a dangling symlink +would not be found by this method when the pattern is an exact match, but +would be found when the pattern contains a wildcard or the recursive +wildcard (``**``). With this change, a dangling symlink will be found in +both cases. + +.. + +.. bpo: 23041 +.. date: 2021-11-07-15-31-25 +.. nonce: 564i32 +.. section: Library + +Add :data:`~csv.QUOTE_STRINGS` and :data:`~csv.QUOTE_NOTNULL` to the suite +of :mod:`csv` module quoting styles. + +.. + +.. bpo: 24964 +.. date: 2021-05-16-14-28-30 +.. nonce: Oa5Ie_ +.. section: Library + +Added :meth:`http.client.HTTPConnection.get_proxy_response_headers` that +provides access to the HTTP headers on a proxy server response to the +``CONNECT`` request. + +.. + +.. bpo: 17258 +.. date: 2020-05-25-12-42-36 +.. nonce: lf2554 +.. section: Library + +:mod:`multiprocessing` now supports stronger HMAC algorithms for +inter-process connection authentication rather than only HMAC-MD5. + +.. + +.. bpo: 39744 +.. date: 2020-02-25-00-43-22 +.. nonce: hgK689 +.. section: Library + +Make :func:`asyncio.subprocess.Process.communicate` close the subprocess's +stdin even when called with ``input=None``. + +.. + +.. bpo: 22708 +.. date: 2018-07-16-14-10-29 +.. nonce: 592iRR +.. section: Library + +http.client CONNECT method tunnel improvements: Use HTTP 1.1 protocol; send +a matching Host: header with CONNECT, if one is not provided; convert IDN +domain names to Punycode. Patch by Michael Handler. + +.. + +.. date: 2023-05-14-12-11-28 +.. gh-issue: 67056 +.. nonce: nVC2Rf +.. section: Documentation + +Document that the effect of registering or unregistering an :mod:`atexit` +cleanup function from within a registered cleanup function is undefined. + +.. + +.. date: 2023-04-26-23-55-31 +.. gh-issue: 103629 +.. nonce: -0reqn +.. section: Documentation + +Mention the new way of typing ``**kwargs`` with ``Unpack`` and ``TypedDict`` +introduced in :pep:`692`. + +.. + +.. date: 2023-04-25-22-58-08 +.. gh-issue: 48241 +.. nonce: l1Gxxh +.. section: Documentation + +Clarifying documentation about the url parameter to urllib.request.urlopen +and urllib.request.Requst needing to be encoded properly. + +.. + +.. date: 2023-03-10-04-59-35 +.. gh-issue: 86094 +.. nonce: zOYdy8 +.. section: Documentation + +Add support for Unicode Path Extra Field in ZipFile. Patch by Yeojin Kim and +Andrea Giudiceandrea + +.. + +.. date: 2023-03-07-23-30-29 +.. gh-issue: 99202 +.. nonce: hhiAJF +.. section: Documentation + +Fix extension type from documentation for compiling in C++20 mode + +.. + +.. date: 2023-05-15-02-22-44 +.. gh-issue: 104494 +.. nonce: Bkrbfn +.. section: Tests + +Update ``test_pack_configure_in`` and ``test_place_configure_in`` for +changes to error message formatting in Tk 8.7. + +.. + +.. date: 2023-05-14-03-00-00 +.. gh-issue: 104461 +.. nonce: Rmex11 +.. section: Tests + +Run test_configure_screen on X11 only, since the ``DISPLAY`` environment +variable and ``-screen`` option for toplevels are not useful on Tk for Win32 +or Aqua. + +.. + +.. date: 2023-04-25-12-19-37 +.. gh-issue: 86275 +.. nonce: -RoLIt +.. section: Tests + +Added property-based tests to the :mod:`zoneinfo` tests, along with stubs +for the ``hypothesis`` interface. (Patch by Paul Ganssle) + +.. + +.. date: 2023-04-08-00-50-23 +.. gh-issue: 103329 +.. nonce: M38tqF +.. section: Tests + +Regression tests for the behaviour of ``unittest.mock.PropertyMock`` were +added. + +.. + +.. date: 2023-03-17-22-00-47 +.. gh-issue: 102795 +.. nonce: z21EoC +.. section: Tests + +fix use of poll in test_epoll's test_control_and_wait + +.. + +.. date: 2022-11-06-18-42-38 +.. gh-issue: 75729 +.. nonce: uGYJrv +.. section: Tests + +Fix the :func:`os.spawn* ` tests failing on Windows when the +working directory or interpreter path contains spaces. + +.. + +.. date: 2023-05-20-16-09-59 +.. gh-issue: 101282 +.. nonce: FvRARb +.. section: Build + +BOLT optimization is now applied to the libpython shared library if building +a shared library. BOLT instrumentation and application settings can now be +influenced via the ``BOLT_INSTRUMENT_FLAGS`` and ``BOLT_APPLY_FLAGS`` +configure variables. + +.. + +.. date: 2023-05-15-09-34-08 +.. gh-issue: 99017 +.. nonce: nToOQu +.. section: Build + +``PYTHON_FOR_REGEN`` now require Python 3.10 or newer. + +.. + +.. date: 2023-05-14-19-00-19 +.. gh-issue: 104490 +.. nonce: 1tA4AF +.. section: Build + +Define ``.PHONY`` / virtual make targets consistently and properly. + +.. + +.. date: 2023-05-04-10-56-14 +.. gh-issue: 104106 +.. nonce: -W9BJS +.. section: Build + +Add gcc fallback of mkfifoat/mknodat for macOS. Patch by Dong-hee Na. + +.. + +.. date: 2023-04-14-10-24-37 +.. gh-issue: 103532 +.. nonce: H1djkd +.. section: Build + +The ``TKINTER_PROTECT_LOADTK`` macro is no longer defined or used in the +``_tkinter`` module. It was previously only defined when building against +Tk 8.4.13 and older, but Tk older than 8.5.12 has been unsupported since +gh-issue-91152. + +.. + +.. date: 2023-02-11-05-31-05 +.. gh-issue: 99069 +.. nonce: X4LDvY +.. section: Build + +Extended workaround defining ``static_assert`` when missing from the libc +headers to all clang and gcc builds. In particular, this fixes building on +macOS <= 10.10. + +.. + +.. date: 2022-12-18-07-24-44 +.. gh-issue: 100220 +.. nonce: BgSV7C +.. section: Build + +Changed the default value of the ``SHELL`` Makefile variable from +``/bin/sh`` to ``/bin/sh -e`` to ensure that complex recipes correctly fail +after an error. Previously, ``make install`` could fail to install some +files and yet return a successful result. + +.. + +.. date: 2022-06-20-15-15-11 +.. gh-issue: 90656 +.. nonce: kFBbKe +.. section: Build + +Add platform triplets for 64-bit LoongArch: + +* loongarch64-linux-gnusf +* loongarch64-linux-gnuf32 +* loongarch64-linux-gnu + +Patch by Zhang Na. + +.. + +.. date: 2023-05-18-22-46-03 +.. gh-issue: 104623 +.. nonce: HJZhm1 +.. section: Windows + +Update Windows installer to use SQLite 3.42.0. + +.. + +.. date: 2023-04-24-15-51-11 +.. gh-issue: 82814 +.. nonce: GI3UkZ +.. section: Windows + +Fix a potential ``[Errno 13] Permission denied`` when using +:func:`shutil.copystat` within Windows Subsystem for Linux (WSL) on a +mounted filesystem by adding ``errno.EACCES`` to the list of ignored errors +within the internal implementation. + +.. + +.. date: 2023-04-12-10-49-21 +.. gh-issue: 103088 +.. nonce: Yjj-qJ +.. section: Windows + +Fix virtual environment :file:`activate` script having incorrect line +endings for Cygwin. + +.. + +.. date: 2023-04-11-09-22-22 +.. gh-issue: 103088 +.. nonce: 6AJEuR +.. section: Windows + +Fixes venvs not working in bash on Windows across different disks + +.. + +.. date: 2023-03-24-11-25-28 +.. gh-issue: 102997 +.. nonce: dredy2 +.. section: Windows + +Update Windows installer to use SQLite 3.41.2. + +.. + +.. date: 2023-03-18-21-38-00 +.. gh-issue: 88013 +.. nonce: Z3loxC +.. section: Windows + +Fixed a bug where :exc:`TypeError` was raised when calling +:func:`ntpath.realpath` with a bytes parameter in some cases. + +.. + +.. date: 2023-05-21-23-54-52 +.. gh-issue: 99834 +.. nonce: 6ANPts +.. section: macOS + +Update macOS installer to Tcl/Tk 8.6.13. + +.. + +.. date: 2023-05-18-22-31-49 +.. gh-issue: 104623 +.. nonce: 6h7Xfx +.. section: macOS + +Update macOS installer to SQLite 3.42.0. + +.. + +.. date: 2023-05-18-08-52-04 +.. gh-issue: 103545 +.. nonce: pi5k2N +.. section: macOS + +Add ``os.PRIO_DARWIN_THREAD``, ``os.PRIO_DARWIN_PROCESS``, +``os.PRIO_DARWIN_BG`` and ``os.PRIO_DARWIN_NONUI``. These can be used with +``os.setpriority`` to run the process at a lower priority and make use of +the efficiency cores on Apple Silicon systems. + +.. + +.. date: 2023-05-04-21-47-59 +.. gh-issue: 104180 +.. nonce: lEJCwd +.. section: macOS + +Support reading SOCKS proxy configuration from macOS System Configuration. +Patch by Sam Schott. + +.. + +.. date: 2023-04-24-18-37-48 +.. gh-issue: 60436 +.. nonce: in-IyF +.. section: macOS + +update curses textbox to additionally handle backspace using the +``curses.ascii.DEL`` key press. + +.. + +.. date: 2023-04-04-13-37-28 +.. gh-issue: 103207 +.. nonce: x0vvQp +.. section: macOS + +Add instructions to the macOS installer welcome display on how to workaround +the macOS 13 Ventura “The installer encountered an error” failure. + +.. + +.. date: 2023-03-24-11-20-47 +.. gh-issue: 102997 +.. nonce: ZgQkbq +.. section: macOS + +Update macOS installer to SQLite 3.41.2. + +.. + +.. date: 2023-05-17-17-32-21 +.. gh-issue: 104499 +.. nonce: hNeqV4 +.. section: IDLE + +Fix completions for Tk Aqua 8.7 (currently blank). + +.. + +.. date: 2023-05-17-15-11-11 +.. gh-issue: 104496 +.. nonce: wjav-y +.. section: IDLE + +About prints both tcl and tk versions if different (expected someday). + +.. + +.. date: 2023-04-30-20-01-18 +.. gh-issue: 88496 +.. nonce: y65vUb +.. section: IDLE + +Fix IDLE test hang on macOS. + +.. + +.. date: 2023-05-11-15-12-11 +.. gh-issue: 104389 +.. nonce: EiOhB3 +.. section: Tools/Demos + +Argument Clinic C converters now accept the ``unused`` keyword, for wrapping +a parameter with :c:macro:`Py_UNUSED`. Patch by Erlend E. Aasland. + +.. + +.. date: 2023-05-18-20-53-05 +.. gh-issue: 101291 +.. nonce: ZBh9aR +.. section: C API + +Added unstable C API for extracting the value of "compact" integers: +:c:func:`PyUnstable_Long_IsCompact` and +:c:func:`PyUnstable_Long_CompactValue`. + +.. + +.. date: 2023-05-02-21-05-54 +.. gh-issue: 104109 +.. nonce: 0tnDZV +.. section: C API + +We've added ``Py_NewInterpreterFromConfig()`` and ``PyInterpreterConfig`` to +the public C-API (but not the stable ABI; not yet at least). The new +function may be used to create a new interpreter with various features +configured. The function was added to support PEP 684 (per-interpreter +GIL). + +.. + +.. date: 2023-04-28-18-04-38 +.. gh-issue: 103968 +.. nonce: EnVvOx +.. section: C API + +:c:func:`PyType_FromSpec` and its variants now allow creating classes whose +metaclass overrides :c:member:`~PyTypeObject.tp_new`. The ``tp_new`` is +ignored. This behavior is deprecated and will be disallowed in 3.14+. The +new :c:func:`PyType_FromMetaclass` already disallows it. + +.. + +.. date: 2023-04-24-10-31-59 +.. gh-issue: 103743 +.. nonce: 2xYA1K +.. section: C API + +Add :c:func:`PyUnstable_Object_GC_NewWithExtraData` function that can be +used to allocate additional memory after an object for data not managed by +Python. + +.. + +.. date: 2023-04-14-23-05-52 +.. gh-issue: 103295 +.. nonce: GRHY1Z +.. section: C API + +Introduced :c:func:`PyUnstable_WritePerfMapEntry`, +:c:func:`PyUnstable_PerfMapState_Init` and +:c:func:`PyUnstable_PerfMapState_Fini`. These allow extension modules (JIT +compilers in particular) to write to perf-map files in a thread safe manner. +The :doc:`../howto/perf_profiling` also uses these APIs to write entries in +the perf-map file. + +.. + +.. date: 2023-04-13-16-54-00 +.. gh-issue: 103509 +.. nonce: A26Qu8 +.. section: C API + +Added C API for extending types whose instance memory layout is opaque: +:c:member:`PyType_Spec.basicsize` can now be zero or negative, +:c:func:`PyObject_GetTypeData` can be used to get subclass-specific data, +and :c:macro:`Py_TPFLAGS_ITEMS_AT_END` can be used to safely extend +variable-size objects. See :pep:`697` for details. + +.. + +.. date: 2023-03-28-12-31-51 +.. gh-issue: 103091 +.. nonce: CzZyaZ +.. section: C API + +Add a new C-API function to eagerly assign a version tag to a PyTypeObject: +``PyUnstable_Type_AssignVersionTag()``. + +.. + +.. date: 2023-02-09-23-09-29 +.. gh-issue: 101408 +.. nonce: _paFIF +.. section: C API + +:c:func:`PyObject_GC_Resize` should calculate preheader size if needed. +Patch by Dong-hee Na. + +.. + +.. date: 2022-10-29-10-13-20 +.. gh-issue: 98836 +.. nonce: Cy5h_z +.. section: C API + +Add support of more formatting options (left aligning, octals, uppercase +hexadecimals, :c:expr:`intmax_t`, :c:expr:`ptrdiff_t`, :c:expr:`wchar_t` C +strings, variable width and precision) in :c:func:`PyUnicode_FromFormat` and +:c:func:`PyUnicode_FromFormatV`. + +.. + +.. date: 2022-09-15-15-21-34 +.. gh-issue: 96803 +.. nonce: ynBKIS +.. section: C API + +Add unstable C-API functions to get the code object, lasti and line number +from the internal ``_PyInterpreterFrame`` in the limited API. The functions +are: + +* ``PyCodeObject * PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame)`` +* ``int PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame)`` +* ``int PyUnstable_InterpreterFrame_GetLine(struct _PyInterpreterFrame *frame)`` diff --git a/Misc/NEWS.d/next/Build/2022-06-20-15-15-11.gh-issue-90656.kFBbKe.rst b/Misc/NEWS.d/next/Build/2022-06-20-15-15-11.gh-issue-90656.kFBbKe.rst deleted file mode 100644 index dfe71a5552070d..00000000000000 --- a/Misc/NEWS.d/next/Build/2022-06-20-15-15-11.gh-issue-90656.kFBbKe.rst +++ /dev/null @@ -1,7 +0,0 @@ -Add platform triplets for 64-bit LoongArch: - -* loongarch64-linux-gnusf -* loongarch64-linux-gnuf32 -* loongarch64-linux-gnu - -Patch by Zhang Na. diff --git a/Misc/NEWS.d/next/Build/2022-12-18-07-24-44.gh-issue-100220.BgSV7C.rst b/Misc/NEWS.d/next/Build/2022-12-18-07-24-44.gh-issue-100220.BgSV7C.rst deleted file mode 100644 index 7135317cd06fa2..00000000000000 --- a/Misc/NEWS.d/next/Build/2022-12-18-07-24-44.gh-issue-100220.BgSV7C.rst +++ /dev/null @@ -1,4 +0,0 @@ -Changed the default value of the ``SHELL`` Makefile variable from ``/bin/sh`` -to ``/bin/sh -e`` to ensure that complex recipes correctly fail after an error. -Previously, ``make install`` could fail to install some files and yet return -a successful result. diff --git a/Misc/NEWS.d/next/Build/2023-02-11-05-31-05.gh-issue-99069.X4LDvY.rst b/Misc/NEWS.d/next/Build/2023-02-11-05-31-05.gh-issue-99069.X4LDvY.rst deleted file mode 100644 index ae9b4d59ca8cec..00000000000000 --- a/Misc/NEWS.d/next/Build/2023-02-11-05-31-05.gh-issue-99069.X4LDvY.rst +++ /dev/null @@ -1 +0,0 @@ -Extended workaround defining ``static_assert`` when missing from the libc headers to all clang and gcc builds. In particular, this fixes building on macOS <= 10.10. diff --git a/Misc/NEWS.d/next/Build/2023-04-14-10-24-37.gh-issue-103532.H1djkd.rst b/Misc/NEWS.d/next/Build/2023-04-14-10-24-37.gh-issue-103532.H1djkd.rst deleted file mode 100644 index 255c9833282c2f..00000000000000 --- a/Misc/NEWS.d/next/Build/2023-04-14-10-24-37.gh-issue-103532.H1djkd.rst +++ /dev/null @@ -1,4 +0,0 @@ -The ``TKINTER_PROTECT_LOADTK`` macro is no longer defined or used in the -``_tkinter`` module. It was previously only defined when building against -Tk 8.4.13 and older, but Tk older than 8.5.12 has been unsupported since -gh-issue-91152. diff --git a/Misc/NEWS.d/next/Build/2023-05-04-10-56-14.gh-issue-104106.-W9BJS.rst b/Misc/NEWS.d/next/Build/2023-05-04-10-56-14.gh-issue-104106.-W9BJS.rst deleted file mode 100644 index 900e5bd61d6033..00000000000000 --- a/Misc/NEWS.d/next/Build/2023-05-04-10-56-14.gh-issue-104106.-W9BJS.rst +++ /dev/null @@ -1 +0,0 @@ -Add gcc fallback of mkfifoat/mknodat for macOS. Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Build/2023-05-14-19-00-19.gh-issue-104490.1tA4AF.rst b/Misc/NEWS.d/next/Build/2023-05-14-19-00-19.gh-issue-104490.1tA4AF.rst deleted file mode 100644 index 1315b5cb3c1969..00000000000000 --- a/Misc/NEWS.d/next/Build/2023-05-14-19-00-19.gh-issue-104490.1tA4AF.rst +++ /dev/null @@ -1 +0,0 @@ -Define ``.PHONY`` / virtual make targets consistently and properly. diff --git a/Misc/NEWS.d/next/Build/2023-05-15-09-34-08.gh-issue-99017.nToOQu.rst b/Misc/NEWS.d/next/Build/2023-05-15-09-34-08.gh-issue-99017.nToOQu.rst deleted file mode 100644 index a3517ac0204b1d..00000000000000 --- a/Misc/NEWS.d/next/Build/2023-05-15-09-34-08.gh-issue-99017.nToOQu.rst +++ /dev/null @@ -1 +0,0 @@ -``PYTHON_FOR_REGEN`` now require Python 3.10 or newer. diff --git a/Misc/NEWS.d/next/Build/2023-05-20-16-09-59.gh-issue-101282.FvRARb.rst b/Misc/NEWS.d/next/Build/2023-05-20-16-09-59.gh-issue-101282.FvRARb.rst deleted file mode 100644 index cc70d47c6c16dd..00000000000000 --- a/Misc/NEWS.d/next/Build/2023-05-20-16-09-59.gh-issue-101282.FvRARb.rst +++ /dev/null @@ -1,4 +0,0 @@ -BOLT optimization is now applied to the libpython shared library if building -a shared library. BOLT instrumentation and application settings can now be -influenced via the ``BOLT_INSTRUMENT_FLAGS`` and ``BOLT_APPLY_FLAGS`` -configure variables. diff --git a/Misc/NEWS.d/next/C API/2022-09-15-15-21-34.gh-issue-96803.ynBKIS.rst b/Misc/NEWS.d/next/C API/2022-09-15-15-21-34.gh-issue-96803.ynBKIS.rst deleted file mode 100644 index 6fc56d2249f581..00000000000000 --- a/Misc/NEWS.d/next/C API/2022-09-15-15-21-34.gh-issue-96803.ynBKIS.rst +++ /dev/null @@ -1,6 +0,0 @@ -Add unstable C-API functions to get the code object, lasti and line number from -the internal ``_PyInterpreterFrame`` in the limited API. The functions are: - -* ``PyCodeObject * PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame)`` -* ``int PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame)`` -* ``int PyUnstable_InterpreterFrame_GetLine(struct _PyInterpreterFrame *frame)`` diff --git a/Misc/NEWS.d/next/C API/2022-10-29-10-13-20.gh-issue-98836.Cy5h_z.rst b/Misc/NEWS.d/next/C API/2022-10-29-10-13-20.gh-issue-98836.Cy5h_z.rst deleted file mode 100644 index e3730eb3f89e6c..00000000000000 --- a/Misc/NEWS.d/next/C API/2022-10-29-10-13-20.gh-issue-98836.Cy5h_z.rst +++ /dev/null @@ -1,4 +0,0 @@ -Add support of more formatting options (left aligning, octals, uppercase -hexadecimals, :c:expr:`intmax_t`, :c:expr:`ptrdiff_t`, :c:expr:`wchar_t` C -strings, variable width and precision) in :c:func:`PyUnicode_FromFormat` and -:c:func:`PyUnicode_FromFormatV`. diff --git a/Misc/NEWS.d/next/C API/2023-02-09-23-09-29.gh-issue-101408._paFIF.rst b/Misc/NEWS.d/next/C API/2023-02-09-23-09-29.gh-issue-101408._paFIF.rst deleted file mode 100644 index 172d66163d42e6..00000000000000 --- a/Misc/NEWS.d/next/C API/2023-02-09-23-09-29.gh-issue-101408._paFIF.rst +++ /dev/null @@ -1,2 +0,0 @@ -:c:func:`PyObject_GC_Resize` should calculate preheader size if needed. -Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/C API/2023-03-28-12-31-51.gh-issue-103091.CzZyaZ.rst b/Misc/NEWS.d/next/C API/2023-03-28-12-31-51.gh-issue-103091.CzZyaZ.rst deleted file mode 100644 index 28c77b6816af87..00000000000000 --- a/Misc/NEWS.d/next/C API/2023-03-28-12-31-51.gh-issue-103091.CzZyaZ.rst +++ /dev/null @@ -1 +0,0 @@ -Add a new C-API function to eagerly assign a version tag to a PyTypeObject: ``PyUnstable_Type_AssignVersionTag()``. diff --git a/Misc/NEWS.d/next/C API/2023-04-13-16-54-00.gh-issue-103509.A26Qu8.rst b/Misc/NEWS.d/next/C API/2023-04-13-16-54-00.gh-issue-103509.A26Qu8.rst deleted file mode 100644 index af630c3aafa940..00000000000000 --- a/Misc/NEWS.d/next/C API/2023-04-13-16-54-00.gh-issue-103509.A26Qu8.rst +++ /dev/null @@ -1,5 +0,0 @@ -Added C API for extending types whose instance memory layout is opaque: -:c:member:`PyType_Spec.basicsize` can now be zero or negative, -:c:func:`PyObject_GetTypeData` can be used to get subclass-specific data, -and :c:macro:`Py_TPFLAGS_ITEMS_AT_END` can be used to safely extend -variable-size objects. See :pep:`697` for details. diff --git a/Misc/NEWS.d/next/C API/2023-04-14-23-05-52.gh-issue-103295.GRHY1Z.rst b/Misc/NEWS.d/next/C API/2023-04-14-23-05-52.gh-issue-103295.GRHY1Z.rst deleted file mode 100644 index 88222239858bc3..00000000000000 --- a/Misc/NEWS.d/next/C API/2023-04-14-23-05-52.gh-issue-103295.GRHY1Z.rst +++ /dev/null @@ -1,5 +0,0 @@ -Introduced :c:func:`PyUnstable_WritePerfMapEntry`, :c:func:`PyUnstable_PerfMapState_Init` and -:c:func:`PyUnstable_PerfMapState_Fini`. These allow extension modules (JIT compilers in -particular) to write to perf-map files in a thread safe manner. The -:doc:`../howto/perf_profiling` also uses these APIs to write -entries in the perf-map file. diff --git a/Misc/NEWS.d/next/C API/2023-04-24-10-31-59.gh-issue-103743.2xYA1K.rst b/Misc/NEWS.d/next/C API/2023-04-24-10-31-59.gh-issue-103743.2xYA1K.rst deleted file mode 100644 index d074350ed3ebbe..00000000000000 --- a/Misc/NEWS.d/next/C API/2023-04-24-10-31-59.gh-issue-103743.2xYA1K.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add :c:func:`PyUnstable_Object_GC_NewWithExtraData` function that can be used to -allocate additional memory after an object for data not managed by Python. diff --git a/Misc/NEWS.d/next/C API/2023-04-28-18-04-38.gh-issue-103968.EnVvOx.rst b/Misc/NEWS.d/next/C API/2023-04-28-18-04-38.gh-issue-103968.EnVvOx.rst deleted file mode 100644 index 5e4270f82afd84..00000000000000 --- a/Misc/NEWS.d/next/C API/2023-04-28-18-04-38.gh-issue-103968.EnVvOx.rst +++ /dev/null @@ -1,4 +0,0 @@ -:c:func:`PyType_FromSpec` and its variants now allow creating classes whose -metaclass overrides :c:member:`~PyTypeObject.tp_new`. The ``tp_new`` is -ignored. This behavior is deprecated and will be disallowed in 3.14+. The -new :c:func:`PyType_FromMetaclass` already disallows it. diff --git a/Misc/NEWS.d/next/C API/2023-05-02-21-05-54.gh-issue-104109.0tnDZV.rst b/Misc/NEWS.d/next/C API/2023-05-02-21-05-54.gh-issue-104109.0tnDZV.rst deleted file mode 100644 index 2ffc0fa81c014a..00000000000000 --- a/Misc/NEWS.d/next/C API/2023-05-02-21-05-54.gh-issue-104109.0tnDZV.rst +++ /dev/null @@ -1,5 +0,0 @@ -We've added ``Py_NewInterpreterFromConfig()`` and ``PyInterpreterConfig`` to -the public C-API (but not the stable ABI; not yet at least). The new -function may be used to create a new interpreter with various features -configured. The function was added to support PEP 684 (per-interpreter -GIL). diff --git a/Misc/NEWS.d/next/C API/2023-05-18-20-53-05.gh-issue-101291.ZBh9aR.rst b/Misc/NEWS.d/next/C API/2023-05-18-20-53-05.gh-issue-101291.ZBh9aR.rst deleted file mode 100644 index 465af3b209a0fc..00000000000000 --- a/Misc/NEWS.d/next/C API/2023-05-18-20-53-05.gh-issue-101291.ZBh9aR.rst +++ /dev/null @@ -1,3 +0,0 @@ -Added unstable C API for extracting the value of "compact" integers: -:c:func:`PyUnstable_Long_IsCompact` and -:c:func:`PyUnstable_Long_CompactValue`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-12-01-12-58-31.bpo-31821.1FNmwk.rst b/Misc/NEWS.d/next/Core and Builtins/2019-12-01-12-58-31.bpo-31821.1FNmwk.rst deleted file mode 100644 index 13c054fdd68276..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2019-12-01-12-58-31.bpo-31821.1FNmwk.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :func:`!pause_reading` to work when called from :func:`!connection_made` in :mod:`asyncio`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-02-11-15-54-40.bpo-39610.fvgsCl.rst b/Misc/NEWS.d/next/Core and Builtins/2020-02-11-15-54-40.bpo-39610.fvgsCl.rst deleted file mode 100644 index d65e0f3db9d6f5..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-02-11-15-54-40.bpo-39610.fvgsCl.rst +++ /dev/null @@ -1,2 +0,0 @@ -``len()`` for 0-dimensional :class:`memoryview`` objects (such as ``memoryview(ctypes.c_uint8(42))``) now raises a :exc:`TypeError`. -Previously this returned ``1``, which was not consistent with ``mem_0d[0]`` raising an :exc:`IndexError``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst deleted file mode 100644 index f9e71bc1344bb3..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst +++ /dev/null @@ -1,3 +0,0 @@ -Complex function calls are now faster and consume no C stack -space. - diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-08-12-36-25.gh-issue-99184.KIaqzz.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-08-12-36-25.gh-issue-99184.KIaqzz.rst deleted file mode 100644 index 80076831badfea..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-08-12-36-25.gh-issue-99184.KIaqzz.rst +++ /dev/null @@ -1,2 +0,0 @@ -Bypass instance attribute access of ``__name__`` in ``repr`` of -:class:`weakref.ref`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-01-30-15-40-29.gh-issue-97933.nUlp3r.rst b/Misc/NEWS.d/next/Core and Builtins/2023-01-30-15-40-29.gh-issue-97933.nUlp3r.rst deleted file mode 100644 index 2eec05cb3ace5c..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-01-30-15-40-29.gh-issue-97933.nUlp3r.rst +++ /dev/null @@ -1,2 +0,0 @@ -:pep:`709`: inline list, dict and set comprehensions to improve performance -and reduce bytecode size. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-07-17-37-00.gh-issue-102500.RUSQhz.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-07-17-37-00.gh-issue-102500.RUSQhz.rst deleted file mode 100644 index e03113ba05cd7d..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-07-17-37-00.gh-issue-102500.RUSQhz.rst +++ /dev/null @@ -1,3 +0,0 @@ -Make the buffer protocol accessible in Python code using the new -``__buffer__`` and ``__release_buffer__`` magic methods. See :pep:`688` for -details. Patch by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-25-23-24-38.gh-issue-88691.2SWBd1.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-25-23-24-38.gh-issue-88691.2SWBd1.rst deleted file mode 100644 index 761d45b0a3a84f..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-25-23-24-38.gh-issue-88691.2SWBd1.rst +++ /dev/null @@ -1 +0,0 @@ -Reduce the number of inline :opcode:`CACHE` entries for :opcode:`CALL`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst deleted file mode 100644 index 631ef4c7890450..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst +++ /dev/null @@ -1 +0,0 @@ -Implement :pep:`669` Low Impact Monitoring for CPython. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-01-00-46-31.gh-issue-102700.493NB4.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-01-00-46-31.gh-issue-102700.493NB4.rst deleted file mode 100644 index 46951486e4f9c9..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-01-00-46-31.gh-issue-102700.493NB4.rst +++ /dev/null @@ -1 +0,0 @@ -Allow built-in modules to be submodules. This allows submodules to be statically linked into a CPython binary. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-02-22-14-57.gh-issue-84436.hvMgwF.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-02-22-14-57.gh-issue-84436.hvMgwF.rst deleted file mode 100644 index c4d8ce75b35a30..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-02-22-14-57.gh-issue-84436.hvMgwF.rst +++ /dev/null @@ -1,3 +0,0 @@ -The implementation of PEP-683 which adds Immortal Objects by using a fixed -reference count that skips reference counting to make objects truly -immutable. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-07-12-18-41.gh-issue-103323.9802br.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-07-12-18-41.gh-issue-103323.9802br.rst deleted file mode 100644 index 347c91d973e5ce..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-07-12-18-41.gh-issue-103323.9802br.rst +++ /dev/null @@ -1,3 +0,0 @@ -We've replaced our use of ``_PyRuntime.tstate_current`` with a thread-local -variable. This is a fairly low-level implementation detail, and there -should be no change in behavior. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-08-17-13-07.gh-issue-103242.ysI1b3.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-08-17-13-07.gh-issue-103242.ysI1b3.rst deleted file mode 100644 index 38b107f3be174f..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-08-17-13-07.gh-issue-103242.ysI1b3.rst +++ /dev/null @@ -1,2 +0,0 @@ -Migrate :meth:`~ssl.SSLContext.set_ecdh_curve` method not to use deprecated -OpenSSL APIs. Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-09-04-30-02.gh-issue-103333.gKOetS.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-09-04-30-02.gh-issue-103333.gKOetS.rst deleted file mode 100644 index 793f02c2afdcff..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-09-04-30-02.gh-issue-103333.gKOetS.rst +++ /dev/null @@ -1 +0,0 @@ -:exc:`AttributeError` now retains the ``name`` attribute when pickled and unpickled. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-09-22-21-57.gh-issue-77757._Ow-u2.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-09-22-21-57.gh-issue-77757._Ow-u2.rst deleted file mode 100644 index 85c8ecf7de8d1b..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-09-22-21-57.gh-issue-77757._Ow-u2.rst +++ /dev/null @@ -1,3 +0,0 @@ -Exceptions raised in a typeobject's ``__set_name__`` method are no longer -wrapped by a :exc:`RuntimeError`. Context information is added to the -exception as a :pep:`678` note. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-12-19-55-24.gh-issue-82012.FlcJAh.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-12-19-55-24.gh-issue-82012.FlcJAh.rst deleted file mode 100644 index 819a2359bf6fae..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-12-19-55-24.gh-issue-82012.FlcJAh.rst +++ /dev/null @@ -1,5 +0,0 @@ -The bitwise inversion operator (``~``) on bool is deprecated. -It returns the bitwise inversion of the underlying ``int`` representation such that -``bool(~True) == True``, which can be confusing. Use ``not`` for logical negation -of bools. In the rare case that you really need the bitwise inversion of the underlying ``int``, -convert to int explicitly ``~int(x)``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-12-20-18-51.gh-issue-103488.vYvlHD.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-12-20-18-51.gh-issue-103488.vYvlHD.rst deleted file mode 100644 index e7daa104e57105..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-12-20-18-51.gh-issue-103488.vYvlHD.rst +++ /dev/null @@ -1,3 +0,0 @@ -Change the internal offset distinguishing yield and return target addresses, -so that the instruction pointer is correct for exception handling and other -stack unwinding. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-12-20-22-03.gh-issue-87729.99A7UO.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-12-20-22-03.gh-issue-87729.99A7UO.rst deleted file mode 100644 index 9d75de1565a170..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-12-20-22-03.gh-issue-87729.99A7UO.rst +++ /dev/null @@ -1,4 +0,0 @@ -Add :opcode:`LOAD_SUPER_ATTR` (and a specialization for ``super().method()``) to -speed up ``super().method()`` and ``super().attr``. This makes -``super().method()`` roughly 2.3x faster and brings it within 20% of the -performance of a simple method call. Patch by Vladimir Matveev and Carl Meyer. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-13-00-58-55.gh-issue-103492.P4k0Ay.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-13-00-58-55.gh-issue-103492.P4k0Ay.rst deleted file mode 100644 index 929650968173e7..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-13-00-58-55.gh-issue-103492.P4k0Ay.rst +++ /dev/null @@ -1 +0,0 @@ -Clarify :exc:`SyntaxWarning` with literal ``is`` comparison by specifying which literal is problematic, since comparisons using ``is`` with e.g. None and bool literals are idiomatic. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-14-22-35-23.gh-issue-101517.5EqM-S.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-14-22-35-23.gh-issue-101517.5EqM-S.rst deleted file mode 100644 index 730c6cd40d7235..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-14-22-35-23.gh-issue-101517.5EqM-S.rst +++ /dev/null @@ -1 +0,0 @@ -Fix bug in line numbers of instructions emitted for :keyword:`except* `. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-16-14-38-39.gh-issue-100530.OR6-sn.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-16-14-38-39.gh-issue-100530.OR6-sn.rst deleted file mode 100644 index 5b1bcc4a680fc3..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-16-14-38-39.gh-issue-100530.OR6-sn.rst +++ /dev/null @@ -1 +0,0 @@ -Clarify the error message raised when the called part of a class pattern isn't actually a class. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-17-16-00-32.gh-issue-102856.UunJ7y.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-17-16-00-32.gh-issue-102856.UunJ7y.rst deleted file mode 100644 index 35eceb83816bcb..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-17-16-00-32.gh-issue-102856.UunJ7y.rst +++ /dev/null @@ -1 +0,0 @@ -Implement the required C tokenizer changes for PEP 701. Patch by Pablo Galindo Salgado, Lysandros Nikolaou, Batuhan Taskaya, Marta Gómez Macías and sunmy2019. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-20-16-17-51.gh-issue-103650.K1MFXR.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-20-16-17-51.gh-issue-103650.K1MFXR.rst deleted file mode 100644 index 5434660e9d6ffb..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-20-16-17-51.gh-issue-103650.K1MFXR.rst +++ /dev/null @@ -1 +0,0 @@ -Change the perf map format to remove the '0x' prefix from the addresses diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-21-16-12-41.gh-issue-103590.7DHDOE.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-21-16-12-41.gh-issue-103590.7DHDOE.rst deleted file mode 100644 index af733a8207a2c1..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-21-16-12-41.gh-issue-103590.7DHDOE.rst +++ /dev/null @@ -1 +0,0 @@ -Do not wrap a single exception raised from a ``try-except*`` construct in an :exc:`ExceptionGroup`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-21-17-03-14.gh-issue-102310.anLjDx.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-21-17-03-14.gh-issue-102310.anLjDx.rst deleted file mode 100644 index 15cb6c64adbab1..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-21-17-03-14.gh-issue-102310.anLjDx.rst +++ /dev/null @@ -1 +0,0 @@ -Change the error range for invalid bytes literals. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-24-14-38-16.gh-issue-103793.kqoH6Q.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-24-14-38-16.gh-issue-103793.kqoH6Q.rst deleted file mode 100644 index c48348798e7142..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-24-14-38-16.gh-issue-103793.kqoH6Q.rst +++ /dev/null @@ -1,3 +0,0 @@ -Optimized asyncio Task creation by deferring expensive string formatting -(task name generation) from Task creation to the first time ``get_name`` is -called. This makes asyncio benchmarks up to 5% faster. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-24-21-47-38.gh-issue-103801.WaBanq.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-24-21-47-38.gh-issue-103801.WaBanq.rst deleted file mode 100644 index 6f07d72fafdfc3..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-24-21-47-38.gh-issue-103801.WaBanq.rst +++ /dev/null @@ -1 +0,0 @@ -Adds three minor linting fixes to the wasm module caught that were caught by ruff. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-25-08-43-11.gh-issue-103763.ZLBZk1.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-25-08-43-11.gh-issue-103763.ZLBZk1.rst deleted file mode 100644 index cb8d2565012658..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-25-08-43-11.gh-issue-103763.ZLBZk1.rst +++ /dev/null @@ -1,23 +0,0 @@ -Implement :pep:`695`, adding syntactic support for generic classes, generic -functions, and type aliases. - -A new ``type X = ...`` syntax is added for type aliases, which resolves at -runtime to an instance of the new class ``typing.TypeAliasType``. -The value is lazily evaluated and is accessible through the ``.__value__`` -attribute. This is implemented as a new AST node ``ast.TypeAlias``. - -New syntax (``class X[T]: ...``, ``def func[T](): ...``) is added for defining -generic functions and classes. This is implemented as a new -``type_params`` attribute on the AST nodes for classes and functions. -This node holds instances of the new AST classes ``ast.TypeVar``, -``ast.ParamSpec``, and ``ast.TypeVarTuple``. - -``typing.TypeVar``, ``typing.ParamSpec``, ``typing.ParamSpecArgs``, -``typing.ParamSpecKwargs``, ``typing.TypeVarTuple``, and -``typing.Generic`` are now implemented in C rather than Python. - -There are new bytecode instructions ``LOAD_LOCALS``, -``LOAD_CLASSDICT_OR_GLOBAL``, and ``LOAD_CLASSDICT_OR_DEREF`` -to support correct resolution of names in class namespaces. - -Patch by Eric Traut, Larry Hastings, and Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-25-20-56-01.gh-issue-103845.V7NYFn.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-25-20-56-01.gh-issue-103845.V7NYFn.rst deleted file mode 100644 index e8434854cde632..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-25-20-56-01.gh-issue-103845.V7NYFn.rst +++ /dev/null @@ -1 +0,0 @@ -Remove both line and instruction instrumentation before adding new ones for monitoring, to avoid newly added instrumentation being removed immediately. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-26-15-14-23.gh-issue-103899.1pqKPF.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-26-15-14-23.gh-issue-103899.1pqKPF.rst deleted file mode 100644 index c12a6b9cb841f2..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-26-15-14-23.gh-issue-103899.1pqKPF.rst +++ /dev/null @@ -1,3 +0,0 @@ -Provide a helpful hint in the :exc:`TypeError` message when accidentally -calling a :term:`module` object that has a callable attribute of the same -name (such as :func:`dis.dis` or :class:`datetime.datetime`). diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-26-16-26-35.gh-issue-103907.kiONZQ.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-26-16-26-35.gh-issue-103907.kiONZQ.rst deleted file mode 100644 index cb2aaffed9ff14..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-26-16-26-35.gh-issue-103907.kiONZQ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Don't modify the refcounts of known immortal objects (:const:`True`, -:const:`False`, and :const:`None`) in the main interpreter loop. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-26-17-56-18.gh-issue-103895.ESB6tn.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-26-17-56-18.gh-issue-103895.ESB6tn.rst deleted file mode 100644 index 6fed304c9132b3..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-26-17-56-18.gh-issue-103895.ESB6tn.rst +++ /dev/null @@ -1,3 +0,0 @@ -Improve handling of edge cases in showing ``Exception.__notes__``. Ensures -that the messages always end with a newline and that string/bytes are not -exploded over multiple lines. Patch by Carey Metcalfe. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-08-08-05.gh-issue-102213.nfH-4C.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-08-08-05.gh-issue-102213.nfH-4C.rst deleted file mode 100644 index 997bef226e713f..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-08-08-05.gh-issue-102213.nfH-4C.rst +++ /dev/null @@ -1 +0,0 @@ -Fix performance loss when accessing an object's attributes with ``__getattr__`` defined. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-12-03-52.gh-issue-104018.PFxGS4.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-12-03-52.gh-issue-104018.PFxGS4.rst deleted file mode 100644 index f3cadaee0e32d9..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-12-03-52.gh-issue-104018.PFxGS4.rst +++ /dev/null @@ -1 +0,0 @@ -Disallow the "z" format specifier in %-format of bytes objects. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-10-38.gh-issue-104028.dxfh13.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-10-38.gh-issue-104028.dxfh13.rst deleted file mode 100644 index 9c35ea88499dce..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-10-38.gh-issue-104028.dxfh13.rst +++ /dev/null @@ -1,2 +0,0 @@ -Reduce object creation while calling callback function from gc. -Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-48-29.gh-issue-104066.pzoUZQ.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-48-29.gh-issue-104066.pzoUZQ.rst deleted file mode 100644 index 97e0c01689cb6f..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-48-29.gh-issue-104066.pzoUZQ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve the performance of :func:`hasattr` for module objects with a missing -attribute. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-21-05-47.gh-issue-104078.vRaBsU.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-21-05-47.gh-issue-104078.vRaBsU.rst deleted file mode 100644 index 6f24529bac3e0c..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-21-05-47.gh-issue-104078.vRaBsU.rst +++ /dev/null @@ -1 +0,0 @@ -Improve the performance of :c:func:`PyObject_HasAttrString` diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-02-18-29-49.gh-issue-104142._5Et6I.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-02-18-29-49.gh-issue-104142._5Et6I.rst deleted file mode 100644 index 6a19ae84057f4c..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-02-18-29-49.gh-issue-104142._5Et6I.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix an issue where :class:`list` or :class:`tuple` repetition could fail to -respect :pep:`683`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-03-17-46-47.gh-issue-104108.GOxAYt.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-03-17-46-47.gh-issue-104108.GOxAYt.rst deleted file mode 100644 index dad843636493ae..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-03-17-46-47.gh-issue-104108.GOxAYt.rst +++ /dev/null @@ -1,6 +0,0 @@ -Multi-phase init extension modules may now indicate whether or not they -actually support multiple interpreters. By default such modules are -expected to support use in multiple interpreters. In the uncommon case that -one does not, it may use the new ``Py_mod_multiple_interpreters`` module def -slot. A value of ``0`` means the module does not support them. ``1`` means -it does. The default is ``1``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-05-12-14-47.gh-issue-99113.-RAdnv.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-05-12-14-47.gh-issue-99113.-RAdnv.rst deleted file mode 100644 index 42e26cb27b6e01..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-05-12-14-47.gh-issue-99113.-RAdnv.rst +++ /dev/null @@ -1,6 +0,0 @@ -The GIL is now (optionally) per-interpreter. This is the fundamental change -for PEP 684. This is all made possible by virtue of the isolated state of -each interpreter in the process. The behavior of the main interpreter -remains unchanged. Likewise, interpreters created using -``Py_NewInterpreter()`` are not affected. To get an interpreter with its -own GIL, call ``Py_NewInterpreterFromConfig()``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-05-13-18-56.gh-issue-99113.hT1ajK.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-05-13-18-56.gh-issue-99113.hT1ajK.rst deleted file mode 100644 index afd26750846167..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-05-13-18-56.gh-issue-99113.hT1ajK.rst +++ /dev/null @@ -1,11 +0,0 @@ -Multi-phase init extension modules may now indicate that they support -running in subinterpreters that have their own GIL. This is done by using -``Py_MOD_PER_INTERPRETER_GIL_SUPPORTED`` as the value for the -``Py_mod_multiple_interpreters`` module def slot. Otherwise the module, by -default, cannot be imported in such subinterpreters. (This does not affect -the main interpreter or subinterpreters that do not have their own GIL.) In -addition to the isolation that multi-phase init already normally requires, -support for per-interpreter GIL involves one additional constraint: -thread-safety. If the module has external (linked) dependencies and those -libraries have any state that isn't thread-safe then the module must do the -additional work to add thread-safety. This should be an uncommon case. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-08-10-34-55.gh-issue-104263.ctHWI8.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-08-10-34-55.gh-issue-104263.ctHWI8.rst deleted file mode 100644 index 342467cfcd4e75..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-08-10-34-55.gh-issue-104263.ctHWI8.rst +++ /dev/null @@ -1,6 +0,0 @@ -Fix ``float("nan")`` to produce a quiet NaN on platforms (like MIPS) where -the meaning of the signalling / quiet bit is inverted from its usual -meaning. Also introduce a new macro ``Py_INFINITY`` matching C99's -``INFINITY``, and refactor internals to rely on C99's ``NAN`` and -``INFINITY`` macros instead of hard-coding bit patterns for infinities and -NaNs. Thanks Sebastian Berg. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-10-20-52-29.gh-issue-103082.y3LG5Q.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-10-20-52-29.gh-issue-103082.y3LG5Q.rst deleted file mode 100644 index 40eee64dfa717d..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-10-20-52-29.gh-issue-103082.y3LG5Q.rst +++ /dev/null @@ -1,5 +0,0 @@ -Change behavior of ``sys.monitoring.events.LINE`` events in -``sys.monitoring``: Line events now occur when a new line is reached -dynamically, instead of using a static approximation, as before. This makes -the behavior very similar to that of "line" events in ``sys.settrace``. This -should ease porting of tools from 3.11 to 3.12. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-11-15-56-07.gh-issue-104405.tXV5fn.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-11-15-56-07.gh-issue-104405.tXV5fn.rst deleted file mode 100644 index 06ec5d7b0f0cc9..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-11-15-56-07.gh-issue-104405.tXV5fn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix an issue where some :term:`bytecode` instructions could ignore -:pep:`523` when "inlining" calls. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-12-13-30-04.gh-issue-102818.rnv1mH.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-12-13-30-04.gh-issue-102818.rnv1mH.rst deleted file mode 100644 index 5cb77bff792c90..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-12-13-30-04.gh-issue-102818.rnv1mH.rst +++ /dev/null @@ -1,5 +0,0 @@ -Do not add a frame to the traceback in the ``sys.setprofile`` and -``sys.settrace`` trampoline functions. This ensures that frames are not -duplicated if an exception is raised in the callback function, and ensures -that frames are not omitted if a C callback is used and that does not add the -frame. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-14-18-56-54.gh-issue-104482.yaQsv8.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-14-18-56-54.gh-issue-104482.yaQsv8.rst deleted file mode 100644 index 07c09a3a84a62c..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-14-18-56-54.gh-issue-104482.yaQsv8.rst +++ /dev/null @@ -1 +0,0 @@ -Fix three error handling bugs in ast.c's validation of pattern matching statements. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-16-19-17-48.gh-issue-104572.eBZQYS.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-16-19-17-48.gh-issue-104572.eBZQYS.rst deleted file mode 100644 index 25bad8aa6e3e6f..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-16-19-17-48.gh-issue-104572.eBZQYS.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve syntax error message for invalid constructs in :pep:`695` contexts -and in annotations when ``from __future__ import annotations`` is active. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-18-13-00-21.gh-issue-104615.h_rtw2.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-18-13-00-21.gh-issue-104615.h_rtw2.rst deleted file mode 100644 index 447291a619dd67..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-18-13-00-21.gh-issue-104615.h_rtw2.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix wrong ordering of assignments in code like ``a, a = x, y``. Contributed by -Carl Meyer. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-20-23-08-48.gh-issue-102856.Knv9WT.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-20-23-08-48.gh-issue-102856.Knv9WT.rst deleted file mode 100644 index ff831c9f935db3..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-20-23-08-48.gh-issue-102856.Knv9WT.rst +++ /dev/null @@ -1 +0,0 @@ -Implement PEP 701 changes in the :mod:`tokenize` module. Patch by Marta Gómez Macías and Pablo Galindo Salgado diff --git a/Misc/NEWS.d/next/Documentation/2023-03-07-23-30-29.gh-issue-99202.hhiAJF.rst b/Misc/NEWS.d/next/Documentation/2023-03-07-23-30-29.gh-issue-99202.hhiAJF.rst deleted file mode 100644 index 1569e815ee50fa..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2023-03-07-23-30-29.gh-issue-99202.hhiAJF.rst +++ /dev/null @@ -1 +0,0 @@ -Fix extension type from documentation for compiling in C++20 mode diff --git a/Misc/NEWS.d/next/Documentation/2023-03-10-04-59-35.gh-issue-86094.zOYdy8.rst b/Misc/NEWS.d/next/Documentation/2023-03-10-04-59-35.gh-issue-86094.zOYdy8.rst deleted file mode 100644 index 39461f3f84c9ac..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2023-03-10-04-59-35.gh-issue-86094.zOYdy8.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add support for Unicode Path Extra Field in ZipFile. Patch by Yeojin Kim -and Andrea Giudiceandrea diff --git a/Misc/NEWS.d/next/Documentation/2023-04-25-22-58-08.gh-issue-48241.l1Gxxh.rst b/Misc/NEWS.d/next/Documentation/2023-04-25-22-58-08.gh-issue-48241.l1Gxxh.rst deleted file mode 100644 index 619505cf6ee5b8..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2023-04-25-22-58-08.gh-issue-48241.l1Gxxh.rst +++ /dev/null @@ -1 +0,0 @@ -Clarifying documentation about the url parameter to urllib.request.urlopen and urllib.request.Requst needing to be encoded properly. diff --git a/Misc/NEWS.d/next/Documentation/2023-04-26-23-55-31.gh-issue-103629.-0reqn.rst b/Misc/NEWS.d/next/Documentation/2023-04-26-23-55-31.gh-issue-103629.-0reqn.rst deleted file mode 100644 index 6dc0a1cb5a3e4f..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2023-04-26-23-55-31.gh-issue-103629.-0reqn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Mention the new way of typing ``**kwargs`` with ``Unpack`` and ``TypedDict`` -introduced in :pep:`692`. diff --git a/Misc/NEWS.d/next/Documentation/2023-05-14-12-11-28.gh-issue-67056.nVC2Rf.rst b/Misc/NEWS.d/next/Documentation/2023-05-14-12-11-28.gh-issue-67056.nVC2Rf.rst deleted file mode 100644 index 2c6ef17810723d..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2023-05-14-12-11-28.gh-issue-67056.nVC2Rf.rst +++ /dev/null @@ -1,2 +0,0 @@ -Document that the effect of registering or unregistering an :mod:`atexit` -cleanup function from within a registered cleanup function is undefined. diff --git a/Misc/NEWS.d/next/IDLE/2023-04-30-20-01-18.gh-issue-88496.y65vUb.rst b/Misc/NEWS.d/next/IDLE/2023-04-30-20-01-18.gh-issue-88496.y65vUb.rst deleted file mode 100644 index 4f390d189d23b5..00000000000000 --- a/Misc/NEWS.d/next/IDLE/2023-04-30-20-01-18.gh-issue-88496.y65vUb.rst +++ /dev/null @@ -1 +0,0 @@ -Fix IDLE test hang on macOS. diff --git a/Misc/NEWS.d/next/IDLE/2023-05-17-15-11-11.gh-issue-104496.wjav-y.rst b/Misc/NEWS.d/next/IDLE/2023-05-17-15-11-11.gh-issue-104496.wjav-y.rst deleted file mode 100644 index 18d83150e94f01..00000000000000 --- a/Misc/NEWS.d/next/IDLE/2023-05-17-15-11-11.gh-issue-104496.wjav-y.rst +++ /dev/null @@ -1 +0,0 @@ -About prints both tcl and tk versions if different (expected someday). diff --git a/Misc/NEWS.d/next/IDLE/2023-05-17-17-32-21.gh-issue-104499.hNeqV4.rst b/Misc/NEWS.d/next/IDLE/2023-05-17-17-32-21.gh-issue-104499.hNeqV4.rst deleted file mode 100644 index 20cbd9ab9e571f..00000000000000 --- a/Misc/NEWS.d/next/IDLE/2023-05-17-17-32-21.gh-issue-104499.hNeqV4.rst +++ /dev/null @@ -1 +0,0 @@ -Fix completions for Tk Aqua 8.7 (currently blank). diff --git a/Misc/NEWS.d/next/Library/2018-07-16-14-10-29.bpo-22708.592iRR.rst b/Misc/NEWS.d/next/Library/2018-07-16-14-10-29.bpo-22708.592iRR.rst deleted file mode 100644 index 00bcf38bbcdf51..00000000000000 --- a/Misc/NEWS.d/next/Library/2018-07-16-14-10-29.bpo-22708.592iRR.rst +++ /dev/null @@ -1,3 +0,0 @@ -http.client CONNECT method tunnel improvements: Use HTTP 1.1 protocol; send -a matching Host: header with CONNECT, if one is not provided; convert IDN -domain names to Punycode. Patch by Michael Handler. diff --git a/Misc/NEWS.d/next/Library/2020-02-25-00-43-22.bpo-39744.hgK689.rst b/Misc/NEWS.d/next/Library/2020-02-25-00-43-22.bpo-39744.hgK689.rst deleted file mode 100644 index 6e690f996569a4..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-02-25-00-43-22.bpo-39744.hgK689.rst +++ /dev/null @@ -1 +0,0 @@ -Make :func:`asyncio.subprocess.Process.communicate` close the subprocess's stdin even when called with ``input=None``. diff --git a/Misc/NEWS.d/next/Library/2020-05-25-12-42-36.bpo-17258.lf2554.rst b/Misc/NEWS.d/next/Library/2020-05-25-12-42-36.bpo-17258.lf2554.rst deleted file mode 100644 index 18ebd6e140cff7..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-05-25-12-42-36.bpo-17258.lf2554.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`multiprocessing` now supports stronger HMAC algorithms for inter-process -connection authentication rather than only HMAC-MD5. diff --git a/Misc/NEWS.d/next/Library/2021-05-16-14-28-30.bpo-24964.Oa5Ie_.rst b/Misc/NEWS.d/next/Library/2021-05-16-14-28-30.bpo-24964.Oa5Ie_.rst deleted file mode 100644 index 0904b162e63523..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-16-14-28-30.bpo-24964.Oa5Ie_.rst +++ /dev/null @@ -1,3 +0,0 @@ -Added :meth:`http.client.HTTPConnection.get_proxy_response_headers` that -provides access to the HTTP headers on a proxy server response to the -``CONNECT`` request. diff --git a/Misc/NEWS.d/next/Library/2021-11-07-15-31-25.bpo-23041.564i32.rst b/Misc/NEWS.d/next/Library/2021-11-07-15-31-25.bpo-23041.564i32.rst deleted file mode 100644 index 53c32d397b206b..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-11-07-15-31-25.bpo-23041.564i32.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add :data:`~csv.QUOTE_STRINGS` and :data:`~csv.QUOTE_NOTNULL` to the suite -of :mod:`csv` module quoting styles. diff --git a/Misc/NEWS.d/next/Library/2021-11-19-23-37-18.bpo-45606.UW5XE1.rst b/Misc/NEWS.d/next/Library/2021-11-19-23-37-18.bpo-45606.UW5XE1.rst deleted file mode 100644 index 531f4729220036..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-11-19-23-37-18.bpo-45606.UW5XE1.rst +++ /dev/null @@ -1,5 +0,0 @@ -Fixed the bug in :meth:`pathlib.Path.glob` -- previously a dangling symlink -would not be found by this method when the pattern is an exact match, but -would be found when the pattern contains a wildcard or the recursive -wildcard (``**``). With this change, a dangling symlink will be found in -both cases. diff --git a/Misc/NEWS.d/next/Library/2021-12-03-23-00-56.bpo-44844.tvg2VY.rst b/Misc/NEWS.d/next/Library/2021-12-03-23-00-56.bpo-44844.tvg2VY.rst deleted file mode 100644 index f0c91236dfdf21..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-12-03-23-00-56.bpo-44844.tvg2VY.rst +++ /dev/null @@ -1 +0,0 @@ -Enables :mod:`webbrowser` to detect and launch Microsoft Edge browser. diff --git a/Misc/NEWS.d/next/Library/2022-02-19-14-19-34.bpo-46797.6BXZX4.rst b/Misc/NEWS.d/next/Library/2022-02-19-14-19-34.bpo-46797.6BXZX4.rst deleted file mode 100644 index 6539efbc9d0eb0..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-02-19-14-19-34.bpo-46797.6BXZX4.rst +++ /dev/null @@ -1,4 +0,0 @@ -Deprecation warnings are now emitted for :class:`!ast.Num`, -:class:`!ast.Bytes`, :class:`!ast.Str`, :class:`!ast.NameConstant` and -:class:`!ast.Ellipsis`. These have been documented as deprecated since Python -3.8, and will be removed in Python 3.14. diff --git a/Misc/NEWS.d/next/Library/2022-05-02-16-21-05.gh-issue-92184.hneGVW.rst b/Misc/NEWS.d/next/Library/2022-05-02-16-21-05.gh-issue-92184.hneGVW.rst deleted file mode 100644 index 65dbdc9f60371e..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-05-02-16-21-05.gh-issue-92184.hneGVW.rst +++ /dev/null @@ -1,3 +0,0 @@ -When creating zip files using :mod:`zipfile`, ``os.altsep``, if not ``None``, -will always be treated as a path separator even when it is not ``/``. -Patch by Carey Metcalfe. diff --git a/Misc/NEWS.d/next/Library/2022-07-03-23-13-28.gh-issue-94518.511Tbh.rst b/Misc/NEWS.d/next/Library/2022-07-03-23-13-28.gh-issue-94518.511Tbh.rst deleted file mode 100644 index 7719b74b8e5ef1..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-07-03-23-13-28.gh-issue-94518.511Tbh.rst +++ /dev/null @@ -1 +0,0 @@ -Convert private :meth:`_posixsubprocess.fork_exec` to use Argument Clinic. diff --git a/Misc/NEWS.d/next/Library/2022-07-06-11-10-37.gh-issue-51574.sveUeD.rst b/Misc/NEWS.d/next/Library/2022-07-06-11-10-37.gh-issue-51574.sveUeD.rst deleted file mode 100644 index 50a3d6a4629182..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-07-06-11-10-37.gh-issue-51574.sveUeD.rst +++ /dev/null @@ -1,2 +0,0 @@ -Make :func:`tempfile.mkdtemp` return absolute paths when its *dir* -parameter is relative. diff --git a/Misc/NEWS.d/next/Library/2022-07-16-17-15-29.gh-issue-94906.C4G8DG.rst b/Misc/NEWS.d/next/Library/2022-07-16-17-15-29.gh-issue-94906.C4G8DG.rst deleted file mode 100644 index 663343371d1b99..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-07-16-17-15-29.gh-issue-94906.C4G8DG.rst +++ /dev/null @@ -1 +0,0 @@ -Support multiple steps in :func:`math.nextafter`. Patch by Shantanu Jain and Matthias Gorgens. diff --git a/Misc/NEWS.d/next/Library/2022-08-27-21-41-41.gh-issue-87474.9X-kxt.rst b/Misc/NEWS.d/next/Library/2022-08-27-21-41-41.gh-issue-87474.9X-kxt.rst deleted file mode 100644 index eeb5308680e8af..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-08-27-21-41-41.gh-issue-87474.9X-kxt.rst +++ /dev/null @@ -1 +0,0 @@ -Fix potential file descriptor leaks in :class:`subprocess.Popen`. diff --git a/Misc/NEWS.d/next/Library/2022-09-03-09-24-02.gh-issue-96534.EU4Oxv.rst b/Misc/NEWS.d/next/Library/2022-09-03-09-24-02.gh-issue-96534.EU4Oxv.rst deleted file mode 100644 index 0497d9eb69163e..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-09-03-09-24-02.gh-issue-96534.EU4Oxv.rst +++ /dev/null @@ -1 +0,0 @@ -Support divert(4) added in FreeBSD 14. diff --git a/Misc/NEWS.d/next/Library/2022-09-07-09-32-07.gh-issue-96522.t73oqp.rst b/Misc/NEWS.d/next/Library/2022-09-07-09-32-07.gh-issue-96522.t73oqp.rst deleted file mode 100644 index 12ed52f97129eb..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-09-07-09-32-07.gh-issue-96522.t73oqp.rst +++ /dev/null @@ -1 +0,0 @@ -Fix potential deadlock in pty.spawn() diff --git a/Misc/NEWS.d/next/Library/2022-10-09-14-47-42.gh-issue-98040.IN3qab.rst b/Misc/NEWS.d/next/Library/2022-10-09-14-47-42.gh-issue-98040.IN3qab.rst deleted file mode 100644 index ac185406844153..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-09-14-47-42.gh-issue-98040.IN3qab.rst +++ /dev/null @@ -1,2 +0,0 @@ -Remove more deprecated importlib APIs: ``find_loader()``, ``find_module()``, -``importlib.abc.Finder``, ``pkgutil.ImpImporter``, ``pkgutil.ImpLoader``. diff --git a/Misc/NEWS.d/next/Library/2022-10-20-14-03-58.gh-issue-94473.pzGX73.rst b/Misc/NEWS.d/next/Library/2022-10-20-14-03-58.gh-issue-94473.pzGX73.rst deleted file mode 100644 index f70722a6f0f69c..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-20-14-03-58.gh-issue-94473.pzGX73.rst +++ /dev/null @@ -1,3 +0,0 @@ -Flatten arguments in :meth:`tkinter.Canvas.coords`. It now accepts not only -``x1, y1, x2, y2, ...`` and ``[x1, y1, x2, y2, ...]``, but also ``(x1, y1), -(x2, y2), ...`` and ``[(x1, y1), (x2, y2), ...]``. diff --git a/Misc/NEWS.d/next/Library/2022-10-21-16-23-31.gh-issue-97850.N46coo.rst b/Misc/NEWS.d/next/Library/2022-10-21-16-23-31.gh-issue-97850.N46coo.rst deleted file mode 100644 index e3297d164fff6d..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-21-16-23-31.gh-issue-97850.N46coo.rst +++ /dev/null @@ -1,2 +0,0 @@ -Deprecate :func:`pkgutil.find_loader` and :func:`pkgutil.get_loader` -in favor of :func:`importlib.util.find_spec`. diff --git a/Misc/NEWS.d/next/Library/2022-10-21-17-20-57.gh-issue-98040.3btbmA.rst b/Misc/NEWS.d/next/Library/2022-10-21-17-20-57.gh-issue-98040.3btbmA.rst deleted file mode 100644 index f67bffcb0ddc6c..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-21-17-20-57.gh-issue-98040.3btbmA.rst +++ /dev/null @@ -1 +0,0 @@ -Remove the long-deprecated ``imp`` module. diff --git a/Misc/NEWS.d/next/Library/2022-11-10-16-26-47.gh-issue-99353.DQFjnt.rst b/Misc/NEWS.d/next/Library/2022-11-10-16-26-47.gh-issue-99353.DQFjnt.rst deleted file mode 100644 index 1ad42d5c9aa53d..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-10-16-26-47.gh-issue-99353.DQFjnt.rst +++ /dev/null @@ -1,3 +0,0 @@ -Respect the :class:`http.client.HTTPConnection` ``.debuglevel`` flag -in :class:`urllib.request.AbstractHTTPHandler` when its constructor -parameter ``debuglevel`` is not set. And do the same for ``*HTTPS*``. diff --git a/Misc/NEWS.d/next/Library/2023-01-14-17-54-56.gh-issue-95299.vUhpKz.rst b/Misc/NEWS.d/next/Library/2023-01-14-17-54-56.gh-issue-95299.vUhpKz.rst deleted file mode 100644 index 29c30848e09a83..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-01-14-17-54-56.gh-issue-95299.vUhpKz.rst +++ /dev/null @@ -1 +0,0 @@ -Remove the bundled setuptools wheel from ``ensurepip``, and stop installing setuptools in environments created by ``venv``. diff --git a/Misc/NEWS.d/next/Library/2023-01-22-14-53-12.gh-issue-89550.c1U23f.rst b/Misc/NEWS.d/next/Library/2023-01-22-14-53-12.gh-issue-89550.c1U23f.rst deleted file mode 100644 index 556db0eae00c0b..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-01-22-14-53-12.gh-issue-89550.c1U23f.rst +++ /dev/null @@ -1,2 +0,0 @@ -Decrease execution time of some :mod:`gzip` file writes by 15% by -adding more appropriate buffering. diff --git a/Misc/NEWS.d/next/Library/2023-02-06-16-45-18.gh-issue-83861.mMbIU3.rst b/Misc/NEWS.d/next/Library/2023-02-06-16-45-18.gh-issue-83861.mMbIU3.rst deleted file mode 100644 index e85e7a4ff2e73a..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-06-16-45-18.gh-issue-83861.mMbIU3.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix datetime.astimezone method return value when invoked on a naive datetime -instance that represents local time falling in a timezone transition gap. -PEP 495 requires that instances with fold=1 produce earlier times than those -with fold=0 in this case. diff --git a/Misc/NEWS.d/next/Library/2023-02-09-22-24-34.gh-issue-101640.oFuEpB.rst b/Misc/NEWS.d/next/Library/2023-02-09-22-24-34.gh-issue-101640.oFuEpB.rst deleted file mode 100644 index 917cf0f97b9e06..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-09-22-24-34.gh-issue-101640.oFuEpB.rst +++ /dev/null @@ -1 +0,0 @@ -:class:`argparse.ArgumentParser` now catches errors when writing messages, such as when :data:`sys.stderr` is ``None``. Patch by Oleg Iarygin. diff --git a/Misc/NEWS.d/next/Library/2023-02-11-15-01-32.gh-issue-101688.kwXmfM.rst b/Misc/NEWS.d/next/Library/2023-02-11-15-01-32.gh-issue-101688.kwXmfM.rst deleted file mode 100644 index 6df69463931494..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-11-15-01-32.gh-issue-101688.kwXmfM.rst +++ /dev/null @@ -1,2 +0,0 @@ -Implement :func:`types.get_original_bases` to provide further introspection -for types. diff --git a/Misc/NEWS.d/next/Library/2023-02-11-21-18-10.gh-issue-85984.nvzOD0.rst b/Misc/NEWS.d/next/Library/2023-02-11-21-18-10.gh-issue-85984.nvzOD0.rst deleted file mode 100644 index 246530472165f7..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-11-21-18-10.gh-issue-85984.nvzOD0.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add :func:`tty.cfmakeraw` and :func:`tty.cfmakecbreak` to :mod:`tty` and -modernize, the behavior of :func:`tty.setraw` and :func:`tty.setcbreak` to use -POSIX.1-2017 Chapter 11 "General Terminal Interface" flag masks by default. diff --git a/Misc/NEWS.d/next/Library/2023-02-17-21-14-40.gh-issue-78079.z3Szr6.rst b/Misc/NEWS.d/next/Library/2023-02-17-21-14-40.gh-issue-78079.z3Szr6.rst deleted file mode 100644 index bbb9ac3e3f8faa..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-17-21-14-40.gh-issue-78079.z3Szr6.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix incorrect normalization of UNC device path roots, and partial UNC share -path roots, in :class:`pathlib.PurePath`. Pathlib no longer appends a -trailing slash to such paths. diff --git a/Misc/NEWS.d/next/Library/2023-02-19-12-37-08.gh-issue-62432.GnBFIB.rst b/Misc/NEWS.d/next/Library/2023-02-19-12-37-08.gh-issue-62432.GnBFIB.rst deleted file mode 100644 index a8d66ea48c3278..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-19-12-37-08.gh-issue-62432.GnBFIB.rst +++ /dev/null @@ -1,3 +0,0 @@ -The :mod:`unittest` runner will now exit with status code 5 if no tests -were run. It is common for test runner misconfiguration to fail to find -any tests, this should be an error. diff --git a/Misc/NEWS.d/next/Library/2023-02-21-14-57-34.gh-issue-102114.uUDQzb.rst b/Misc/NEWS.d/next/Library/2023-02-21-14-57-34.gh-issue-102114.uUDQzb.rst deleted file mode 100644 index 4140c9a96cd272..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-21-14-57-34.gh-issue-102114.uUDQzb.rst +++ /dev/null @@ -1 +0,0 @@ -Functions in the :mod:`dis` module that accept a source code string as argument now print a more concise traceback when the string contains a syntax or indentation error. diff --git a/Misc/NEWS.d/next/Library/2023-03-06-18-49-57.gh-issue-101362.eSSy6L.rst b/Misc/NEWS.d/next/Library/2023-03-06-18-49-57.gh-issue-101362.eSSy6L.rst deleted file mode 100644 index 87617a503c0dba..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-06-18-49-57.gh-issue-101362.eSSy6L.rst +++ /dev/null @@ -1,2 +0,0 @@ -Speed up :class:`pathlib.Path` construction by omitting the path anchor from -the internal list of path parts. diff --git a/Misc/NEWS.d/next/Library/2023-03-08-02-45-46.gh-issue-91896.kgON_a.rst b/Misc/NEWS.d/next/Library/2023-03-08-02-45-46.gh-issue-91896.kgON_a.rst deleted file mode 100644 index b5282d3d612916..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-08-02-45-46.gh-issue-91896.kgON_a.rst +++ /dev/null @@ -1 +0,0 @@ -Deprecate :class:`collections.abc.ByteString` diff --git a/Misc/NEWS.d/next/Library/2023-03-14-11-20-19.gh-issue-101819.0-h0it.rst b/Misc/NEWS.d/next/Library/2023-03-14-11-20-19.gh-issue-101819.0-h0it.rst deleted file mode 100644 index 4a73bbf32b370e..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-14-11-20-19.gh-issue-101819.0-h0it.rst +++ /dev/null @@ -1,2 +0,0 @@ -Isolate the :mod:`io` extension module by applying :pep:`687`. Patch by -Kumar Aditya, Victor Stinner, and Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2023-03-15-00-37-43.gh-issue-81079.heTAod.rst b/Misc/NEWS.d/next/Library/2023-03-15-00-37-43.gh-issue-81079.heTAod.rst deleted file mode 100644 index ef5690533985d5..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-15-00-37-43.gh-issue-81079.heTAod.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add *case_sensitive* keyword-only argument to :meth:`pathlib.Path.glob` and -:meth:`~pathlib.Path.rglob`. diff --git a/Misc/NEWS.d/next/Library/2023-03-15-12-18-07.gh-issue-97696.DtnpIC.rst b/Misc/NEWS.d/next/Library/2023-03-15-12-18-07.gh-issue-97696.DtnpIC.rst deleted file mode 100644 index 0b3854d74eb991..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-15-12-18-07.gh-issue-97696.DtnpIC.rst +++ /dev/null @@ -1,6 +0,0 @@ -Implemented an eager task factory in asyncio. -When used as a task factory on an event loop, it performs eager execution of -coroutines. Coroutines that are able to complete synchronously (e.g. return or -raise without blocking) are returned immediately as a finished task, and the -task is never scheduled to the event loop. If the coroutine blocks, the -(pending) task is scheduled and returned. diff --git a/Misc/NEWS.d/next/Library/2023-03-23-15-24-38.gh-issue-102953.YR4KaK.rst b/Misc/NEWS.d/next/Library/2023-03-23-15-24-38.gh-issue-102953.YR4KaK.rst deleted file mode 100644 index 48a105a4a17b29..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-23-15-24-38.gh-issue-102953.YR4KaK.rst +++ /dev/null @@ -1,4 +0,0 @@ -The extraction methods in :mod:`tarfile`, and :func:`shutil.unpack_archive`, -have a new a *filter* argument that allows limiting tar features than may be -surprising or dangerous, such as creating files outside the destination -directory. See :ref:`tarfile-extraction-filter` for details. diff --git a/Misc/NEWS.d/next/Library/2023-03-24-20-49-48.gh-issue-103000.6eVNZI.rst b/Misc/NEWS.d/next/Library/2023-03-24-20-49-48.gh-issue-103000.6eVNZI.rst deleted file mode 100644 index 15f16d9eb4c1bf..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-24-20-49-48.gh-issue-103000.6eVNZI.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve performance of :func:`dataclasses.astuple` and -:func:`dataclasses.asdict` in cases where the contents are common Python types. diff --git a/Misc/NEWS.d/next/Library/2023-03-28-09-13-31.gh-issue-103015.ETTfNf.rst b/Misc/NEWS.d/next/Library/2023-03-28-09-13-31.gh-issue-103015.ETTfNf.rst deleted file mode 100644 index dcac1a28ca5847..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-28-09-13-31.gh-issue-103015.ETTfNf.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add *entrypoint* keyword-only parameter to -:meth:`sqlite3.Connection.load_extension`, for overriding the SQLite -extension entry point. Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2023-03-31-01-13-00.gh-issue-103143.6eMluy.rst b/Misc/NEWS.d/next/Library/2023-03-31-01-13-00.gh-issue-103143.6eMluy.rst deleted file mode 100644 index 32bd62d27c7c6d..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-31-01-13-00.gh-issue-103143.6eMluy.rst +++ /dev/null @@ -1 +0,0 @@ -Polish the help messages and docstrings of :mod:`pdb`. diff --git a/Misc/NEWS.d/next/Library/2023-04-01-23-01-31.gh-issue-103176.FBsdxa.rst b/Misc/NEWS.d/next/Library/2023-04-01-23-01-31.gh-issue-103176.FBsdxa.rst deleted file mode 100644 index b89f9bae595457..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-01-23-01-31.gh-issue-103176.FBsdxa.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`sys._current_exceptions` now returns a mapping from thread-id to an -exception instance, rather than to a ``(typ, exc, tb)`` tuple. diff --git a/Misc/NEWS.d/next/Library/2023-04-02-17-51-08.gh-issue-103193.xrZbM1.rst b/Misc/NEWS.d/next/Library/2023-04-02-17-51-08.gh-issue-103193.xrZbM1.rst deleted file mode 100644 index f0b76a605a5610..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-02-17-51-08.gh-issue-103193.xrZbM1.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve performance of :func:`inspect.getattr_static`. Patch by Alex -Waygood. diff --git a/Misc/NEWS.d/next/Library/2023-04-02-22-04-26.gh-issue-75586.526iJm.rst b/Misc/NEWS.d/next/Library/2023-04-02-22-04-26.gh-issue-75586.526iJm.rst deleted file mode 100644 index 8ec568ec4e4775..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-02-22-04-26.gh-issue-75586.526iJm.rst +++ /dev/null @@ -1 +0,0 @@ -Fix various Windows-specific issues with ``shutil.which``. diff --git a/Misc/NEWS.d/next/Library/2023-04-02-23-05-22.gh-issue-103204.bbDmu0.rst b/Misc/NEWS.d/next/Library/2023-04-02-23-05-22.gh-issue-103204.bbDmu0.rst deleted file mode 100644 index f8b3aa5151b622..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-02-23-05-22.gh-issue-103204.bbDmu0.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fixes :mod:`http.server` accepting HTTP requests with HTTP version numbers -preceded by '+', or '-', or with digit-separating '_' characters. The length -of the version numbers is also constrained. diff --git a/Misc/NEWS.d/next/Library/2023-04-03-21-08-53.gh-issue-103220.OW_Bj5.rst b/Misc/NEWS.d/next/Library/2023-04-03-21-08-53.gh-issue-103220.OW_Bj5.rst deleted file mode 100644 index 9cf26c26873b2a..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-03-21-08-53.gh-issue-103220.OW_Bj5.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix issue where :func:`os.path.join` added a slash when joining onto an -incomplete UNC drive with a trailing slash on Windows. diff --git a/Misc/NEWS.d/next/Library/2023-04-03-22-02-35.gh-issue-100479.kNBjQm.rst b/Misc/NEWS.d/next/Library/2023-04-03-22-02-35.gh-issue-100479.kNBjQm.rst deleted file mode 100644 index 58db90480d2ff0..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-03-22-02-35.gh-issue-100479.kNBjQm.rst +++ /dev/null @@ -1,4 +0,0 @@ -Add :meth:`pathlib.PurePath.with_segments`, which creates a path object from -arguments. This method is called whenever a derivative path is created, such -as from :attr:`pathlib.PurePath.parent`. Subclasses may override this method -to share information between path objects. diff --git a/Misc/NEWS.d/next/Library/2023-04-03-23-43-12.gh-issue-103092.3xqk4y.rst b/Misc/NEWS.d/next/Library/2023-04-03-23-43-12.gh-issue-103092.3xqk4y.rst deleted file mode 100644 index e7586a223c1415..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-03-23-43-12.gh-issue-103092.3xqk4y.rst +++ /dev/null @@ -1 +0,0 @@ -Isolate :mod:`!_socket` (apply :pep:`687`). Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2023-04-03-23-44-34.gh-issue-102978.gy9eVk.rst b/Misc/NEWS.d/next/Library/2023-04-03-23-44-34.gh-issue-102978.gy9eVk.rst deleted file mode 100644 index df63af10a385eb..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-03-23-44-34.gh-issue-102978.gy9eVk.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fixes :func:`unittest.mock.patch` not enforcing function signatures for methods -decorated with ``@classmethod`` or ``@staticmethod`` when patch is called with -``autospec=True``. diff --git a/Misc/NEWS.d/next/Library/2023-04-04-12-43-38.gh-issue-93910.jurMzv.rst b/Misc/NEWS.d/next/Library/2023-04-04-12-43-38.gh-issue-93910.jurMzv.rst deleted file mode 100644 index 783aefae0770a9..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-04-12-43-38.gh-issue-93910.jurMzv.rst +++ /dev/null @@ -1 +0,0 @@ -Remove deprecation of enum ``memmber.member`` access. diff --git a/Misc/NEWS.d/next/Library/2023-04-04-21-27-51.gh-issue-103092.7s7Bzf.rst b/Misc/NEWS.d/next/Library/2023-04-04-21-27-51.gh-issue-103092.7s7Bzf.rst deleted file mode 100644 index 39c62ffbe8c659..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-04-21-27-51.gh-issue-103092.7s7Bzf.rst +++ /dev/null @@ -1 +0,0 @@ -Adapt the :mod:`winsound` extension module to :pep:`687`. diff --git a/Misc/NEWS.d/next/Library/2023-04-04-21-44-25.gh-issue-103092.Dz0_Xn.rst b/Misc/NEWS.d/next/Library/2023-04-04-21-44-25.gh-issue-103092.Dz0_Xn.rst deleted file mode 100644 index 7bd191e3c22b2b..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-04-21-44-25.gh-issue-103092.Dz0_Xn.rst +++ /dev/null @@ -1 +0,0 @@ -Adapt the :mod:`msvcrt` extension module to :pep:`687`. diff --git a/Misc/NEWS.d/next/Library/2023-04-05-01-28-53.gh-issue-103225.QD3JVU.rst b/Misc/NEWS.d/next/Library/2023-04-05-01-28-53.gh-issue-103225.QD3JVU.rst deleted file mode 100644 index 5d1a063acdeb8c..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-05-01-28-53.gh-issue-103225.QD3JVU.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a bug in :mod:`pdb` when displaying line numbers of module-level source code. diff --git a/Misc/NEWS.d/next/Library/2023-04-06-04-35-59.gh-issue-103285.rCZ9-G.rst b/Misc/NEWS.d/next/Library/2023-04-06-04-35-59.gh-issue-103285.rCZ9-G.rst deleted file mode 100644 index 62b4364c2b1665..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-06-04-35-59.gh-issue-103285.rCZ9-G.rst +++ /dev/null @@ -1 +0,0 @@ -Improve performance of :func:`ast.get_source_segment`. diff --git a/Misc/NEWS.d/next/Library/2023-04-06-16-55-51.gh-issue-102778.BWeAmE.rst b/Misc/NEWS.d/next/Library/2023-04-06-16-55-51.gh-issue-102778.BWeAmE.rst deleted file mode 100644 index 64ae5b5b6d564b..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-06-16-55-51.gh-issue-102778.BWeAmE.rst +++ /dev/null @@ -1 +0,0 @@ -Support ``sys.last_exc`` in :mod:`idlelib`. diff --git a/Misc/NEWS.d/next/Library/2023-04-06-17-28-36.gh-issue-103256.1syxfs.rst b/Misc/NEWS.d/next/Library/2023-04-06-17-28-36.gh-issue-103256.1syxfs.rst deleted file mode 100644 index 894c046dcdf0fd..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-06-17-28-36.gh-issue-103256.1syxfs.rst +++ /dev/null @@ -1,6 +0,0 @@ -Fixed a bug that caused :mod:`hmac` to raise an exception when the requested -hash algorithm was not available in OpenSSL despite being available -separately as part of ``hashlib`` itself. It now falls back properly to the -built-in. This could happen when, for example, your OpenSSL does not include -SHA3 support and you want to compute ``hmac.digest(b'K', b'M', -'sha3_256')``. diff --git a/Misc/NEWS.d/next/Library/2023-04-07-15-09-26.gh-issue-74690.0f886b.rst b/Misc/NEWS.d/next/Library/2023-04-07-15-09-26.gh-issue-74690.0f886b.rst deleted file mode 100644 index 0a103ae11970d4..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-07-15-09-26.gh-issue-74690.0f886b.rst +++ /dev/null @@ -1,3 +0,0 @@ -The members of a runtime-checkable protocol are now considered "frozen" at -runtime as soon as the class has been created. See -:ref:`"What's new in Python 3.12" ` for more details. diff --git a/Misc/NEWS.d/next/Library/2023-04-07-15-15-40.gh-issue-74690.un84hh.rst b/Misc/NEWS.d/next/Library/2023-04-07-15-15-40.gh-issue-74690.un84hh.rst deleted file mode 100644 index 48f11aac692ddb..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-07-15-15-40.gh-issue-74690.un84hh.rst +++ /dev/null @@ -1,8 +0,0 @@ -The performance of :func:`isinstance` checks against -:func:`runtime-checkable protocols ` has been -considerably improved for protocols that only have a few members. To achieve -this improvement, several internal implementation details of the -:mod:`typing` module have been refactored, including -``typing._ProtocolMeta.__instancecheck__``, -``typing._is_callable_members_only``, and ``typing._get_protocol_attrs``. -Patches by Alex Waygood. diff --git a/Misc/NEWS.d/next/Library/2023-04-08-00-48-40.gh-issue-103092.5EFts0.rst b/Misc/NEWS.d/next/Library/2023-04-08-00-48-40.gh-issue-103092.5EFts0.rst deleted file mode 100644 index 0f2108fee763d0..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-08-00-48-40.gh-issue-103092.5EFts0.rst +++ /dev/null @@ -1 +0,0 @@ -Adapt the :mod:`winreg` extension module to :pep:`687`. diff --git a/Misc/NEWS.d/next/Library/2023-04-08-01-33-12.gh-issue-103357.vjin28.rst b/Misc/NEWS.d/next/Library/2023-04-08-01-33-12.gh-issue-103357.vjin28.rst deleted file mode 100644 index 83dce56ed0b7c5..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-08-01-33-12.gh-issue-103357.vjin28.rst +++ /dev/null @@ -1,3 +0,0 @@ -Added support for :class:`logging.Formatter` ``defaults`` parameter to -:func:`logging.config.dictConfig` and :func:`logging.config.fileConfig`. -Patch by Bar Harel. diff --git a/Misc/NEWS.d/next/Library/2023-04-09-06-59-36.gh-issue-103092.vskbro.rst b/Misc/NEWS.d/next/Library/2023-04-09-06-59-36.gh-issue-103092.vskbro.rst deleted file mode 100644 index 6977c1489a29cb..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-09-06-59-36.gh-issue-103092.vskbro.rst +++ /dev/null @@ -1 +0,0 @@ -Isolate :mod:`!_collections` (apply :pep:`687`). Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2023-04-11-21-38-39.gh-issue-103449.-nxmhb.rst b/Misc/NEWS.d/next/Library/2023-04-11-21-38-39.gh-issue-103449.-nxmhb.rst deleted file mode 100644 index 0b2b47af1cbaab..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-11-21-38-39.gh-issue-103449.-nxmhb.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a bug in doc string generation in :func:`dataclasses.dataclass`. diff --git a/Misc/NEWS.d/next/Library/2023-04-12-06-00-02.gh-issue-103462.w6yBlM.rst b/Misc/NEWS.d/next/Library/2023-04-12-06-00-02.gh-issue-103462.w6yBlM.rst deleted file mode 100644 index 50758c89cc2856..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-12-06-00-02.gh-issue-103462.w6yBlM.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fixed an issue with using :meth:`~asyncio.WriteTransport.writelines` in :mod:`asyncio` to send very -large payloads that exceed the amount of data that can be written in one -call to :meth:`socket.socket.send` or :meth:`socket.socket.sendmsg`, -resulting in the remaining buffer being left unwritten. diff --git a/Misc/NEWS.d/next/Library/2023-04-12-13-04-16.gh-issue-103472.C6bOHv.rst b/Misc/NEWS.d/next/Library/2023-04-12-13-04-16.gh-issue-103472.C6bOHv.rst deleted file mode 100644 index 01d84f024bd4a6..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-12-13-04-16.gh-issue-103472.C6bOHv.rst +++ /dev/null @@ -1,2 +0,0 @@ -Avoid a potential :exc:`ResourceWarning` in :class:`http.client.HTTPConnection` -by closing the proxy / tunnel's CONNECT response explicitly. diff --git a/Misc/NEWS.d/next/Library/2023-04-12-17-59-55.gh-issue-103365.UBEE0U.rst b/Misc/NEWS.d/next/Library/2023-04-12-17-59-55.gh-issue-103365.UBEE0U.rst deleted file mode 100644 index 4d69f6f6fff713..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-12-17-59-55.gh-issue-103365.UBEE0U.rst +++ /dev/null @@ -1 +0,0 @@ -Set default Flag boundary to ``STRICT`` and fix bitwise operations. diff --git a/Misc/NEWS.d/next/Library/2023-04-13-13-17-47.gh-issue-103489.ZSZgmu.rst b/Misc/NEWS.d/next/Library/2023-04-13-13-17-47.gh-issue-103489.ZSZgmu.rst deleted file mode 100644 index 264564d018ceb4..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-13-13-17-47.gh-issue-103489.ZSZgmu.rst +++ /dev/null @@ -1,4 +0,0 @@ -Add :meth:`~sqlite3.Connection.getconfig` and -:meth:`~sqlite3.Connection.setconfig` to :class:`~sqlite3.Connection` to -make configuration changes to a database connection. Patch by Erlend E. -Aasland. diff --git a/Misc/NEWS.d/next/Library/2023-04-13-19-43-15.gh-issue-103525.uY4VYg.rst b/Misc/NEWS.d/next/Library/2023-04-13-19-43-15.gh-issue-103525.uY4VYg.rst deleted file mode 100644 index 1414cb07dd9155..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-13-19-43-15.gh-issue-103525.uY4VYg.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix misleading exception message when mixed ``str`` and ``bytes`` arguments -are supplied to :class:`pathlib.PurePath` and :class:`~pathlib.Path`. diff --git a/Misc/NEWS.d/next/Library/2023-04-14-06-32-54.gh-issue-103533.n_AfcS.rst b/Misc/NEWS.d/next/Library/2023-04-14-06-32-54.gh-issue-103533.n_AfcS.rst deleted file mode 100644 index 1008ea076c71a0..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-14-06-32-54.gh-issue-103533.n_AfcS.rst +++ /dev/null @@ -1 +0,0 @@ -Update :mod:`cProfile` to use PEP 669 API diff --git a/Misc/NEWS.d/next/Library/2023-04-14-21-12-32.gh-issue-103538.M4FK_v.rst b/Misc/NEWS.d/next/Library/2023-04-14-21-12-32.gh-issue-103538.M4FK_v.rst deleted file mode 100644 index 32788307d6f33b..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-14-21-12-32.gh-issue-103538.M4FK_v.rst +++ /dev/null @@ -1,3 +0,0 @@ -Remove ``_tkinter`` module code guarded by definition of the ``TK_AQUA`` macro -which was only needed for Tk 8.4.7 or earlier and was never actually defined by -any build system or documented for manual use. diff --git a/Misc/NEWS.d/next/Library/2023-04-14-21-16-05.gh-issue-103548.lagdpp.rst b/Misc/NEWS.d/next/Library/2023-04-14-21-16-05.gh-issue-103548.lagdpp.rst deleted file mode 100644 index 238f2868867472..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-14-21-16-05.gh-issue-103548.lagdpp.rst +++ /dev/null @@ -1,4 +0,0 @@ -Improve performance of :meth:`pathlib.Path.absolute` and -:meth:`~pathlib.Path.cwd` by joining paths only when necessary. Also improve -performance of :meth:`pathlib.PurePath.is_absolute` on Posix by skipping path -parsing and normalization. diff --git a/Misc/NEWS.d/next/Library/2023-04-15-11-21-38.gh-issue-103559.a9rYHG.rst b/Misc/NEWS.d/next/Library/2023-04-15-11-21-38.gh-issue-103559.a9rYHG.rst deleted file mode 100644 index 2c9d67e2c4bf71..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-15-11-21-38.gh-issue-103559.a9rYHG.rst +++ /dev/null @@ -1 +0,0 @@ -Update the bundled copy of pip to version 23.1.1. diff --git a/Misc/NEWS.d/next/Library/2023-04-15-12-19-14.gh-issue-103556.TEf-2m.rst b/Misc/NEWS.d/next/Library/2023-04-15-12-19-14.gh-issue-103556.TEf-2m.rst deleted file mode 100644 index fe2267b7b79019..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-15-12-19-14.gh-issue-103556.TEf-2m.rst +++ /dev/null @@ -1,3 +0,0 @@ -Now creating :class:`inspect.Signature` objects with positional-only -parameter with a default followed by a positional-or-keyword parameter -without one is impossible. diff --git a/Misc/NEWS.d/next/Library/2023-04-16-18-29-04.gh-issue-103578.fly1wc.rst b/Misc/NEWS.d/next/Library/2023-04-16-18-29-04.gh-issue-103578.fly1wc.rst deleted file mode 100644 index 69986c2a15b39e..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-16-18-29-04.gh-issue-103578.fly1wc.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed a bug where :mod:`pdb` crashes when reading source file with different encoding by replacing :func:`io.open` with :func:`io.open_code`. The new method would also call into the hook set by :func:`PyFile_SetOpenCodeHook`. diff --git a/Misc/NEWS.d/next/Library/2023-04-16-19-48-21.gh-issue-103584.3mBTuM.rst b/Misc/NEWS.d/next/Library/2023-04-16-19-48-21.gh-issue-103584.3mBTuM.rst deleted file mode 100644 index 6d7c93ade9cd94..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-16-19-48-21.gh-issue-103584.3mBTuM.rst +++ /dev/null @@ -1,12 +0,0 @@ -Updated ``importlib.metadata`` with changes from ``importlib_metadata`` 5.2 -through 6.5.0, including: Support ``installed-files.txt`` for -``Distribution.files`` when present. ``PackageMetadata`` now stipulates an -additional ``get`` method allowing for easy querying of metadata keys that -may not be present. ``packages_distributions`` now honors packages and -modules with Python modules that not ``.py`` sources (e.g. ``.pyc``, -``.so``). Expand protocol for ``PackageMetadata.get_all`` to match the -upstream implementation of ``email.message.Message.get_all`` in -python/typeshed#9620. Deprecated use of ``Distribution`` without defining -abstract methods. Deprecated expectation that -``PackageMetadata.__getitem__`` will return ``None`` for missing keys. In -the future, it will raise a ``KeyError``. diff --git a/Misc/NEWS.d/next/Library/2023-04-17-14-47-28.gh-issue-103596.ME1y3_.rst b/Misc/NEWS.d/next/Library/2023-04-17-14-47-28.gh-issue-103596.ME1y3_.rst deleted file mode 100644 index 2fa27e60b58efe..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-17-14-47-28.gh-issue-103596.ME1y3_.rst +++ /dev/null @@ -1,2 +0,0 @@ -Attributes/methods are no longer shadowed by same-named enum members, -although they may be shadowed by enum.property's. diff --git a/Misc/NEWS.d/next/Library/2023-04-19-16-08-53.gh-issue-84976.HwbzlD.rst b/Misc/NEWS.d/next/Library/2023-04-19-16-08-53.gh-issue-84976.HwbzlD.rst deleted file mode 100644 index 8658627aeba434..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-19-16-08-53.gh-issue-84976.HwbzlD.rst +++ /dev/null @@ -1,5 +0,0 @@ -Create a new ``Lib/_pydatetime.py`` file that defines the Python version of -the ``datetime`` module, and make ``datetime`` import the contents of the -new library only if the C implementation is missing. Currently, the full -Python implementation is defined and then deleted if the C implementation is -not available, slowing down ``import datetime`` unnecessarily. diff --git a/Misc/NEWS.d/next/Library/2023-04-21-10-25-39.gh-issue-103636.YK6NEa.rst b/Misc/NEWS.d/next/Library/2023-04-21-10-25-39.gh-issue-103636.YK6NEa.rst deleted file mode 100644 index b3b5085250f078..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-21-10-25-39.gh-issue-103636.YK6NEa.rst +++ /dev/null @@ -1 +0,0 @@ -Added Enum for months and days in the calendar module. diff --git a/Misc/NEWS.d/next/Library/2023-04-22-02-41-06.gh-issue-103673.oE7S_k.rst b/Misc/NEWS.d/next/Library/2023-04-22-02-41-06.gh-issue-103673.oE7S_k.rst deleted file mode 100644 index bd5317744ff140..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-22-02-41-06.gh-issue-103673.oE7S_k.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`socketserver` gains ``ForkingUnixStreamServer`` and -``ForkingUnixDatagramServer`` classes. Patch by Jay Berry. diff --git a/Misc/NEWS.d/next/Library/2023-04-22-11-20-27.gh-issue-89415.YHk760.rst b/Misc/NEWS.d/next/Library/2023-04-22-11-20-27.gh-issue-89415.YHk760.rst deleted file mode 100644 index a5b99a2f1360f0..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-22-11-20-27.gh-issue-89415.YHk760.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add :mod:`socket` constants for source-specific multicast. -Patch by Reese Hyde. diff --git a/Misc/NEWS.d/next/Library/2023-04-22-12-30-10.gh-issue-92248.NcVTKR.rst b/Misc/NEWS.d/next/Library/2023-04-22-12-30-10.gh-issue-92248.NcVTKR.rst deleted file mode 100644 index d4a02d82941927..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-22-12-30-10.gh-issue-92248.NcVTKR.rst +++ /dev/null @@ -1,2 +0,0 @@ -Deprecate ``type``, ``choices``, and ``metavar`` parameters of -``argparse.BooleanOptionalAction``. diff --git a/Misc/NEWS.d/next/Library/2023-04-22-21-34-13.gh-issue-103693.SBtuLQ.rst b/Misc/NEWS.d/next/Library/2023-04-22-21-34-13.gh-issue-103693.SBtuLQ.rst deleted file mode 100644 index 52c68bfc9ceea4..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-22-21-34-13.gh-issue-103693.SBtuLQ.rst +++ /dev/null @@ -1 +0,0 @@ -Add convenience variable feature to :mod:`pdb` diff --git a/Misc/NEWS.d/next/Library/2023-04-22-22-37-39.gh-issue-103699.NizCjc.rst b/Misc/NEWS.d/next/Library/2023-04-22-22-37-39.gh-issue-103699.NizCjc.rst deleted file mode 100644 index 60547a25a109bc..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-22-22-37-39.gh-issue-103699.NizCjc.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add ``__orig_bases__`` to non-generic TypedDicts, call-based TypedDicts, and -call-based NamedTuples. Other TypedDicts and NamedTuples already had the attribute. diff --git a/Misc/NEWS.d/next/Library/2023-04-23-15-39-17.gh-issue-81403.zVz9Td.rst b/Misc/NEWS.d/next/Library/2023-04-23-15-39-17.gh-issue-81403.zVz9Td.rst deleted file mode 100644 index 6adb71f7677229..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-23-15-39-17.gh-issue-81403.zVz9Td.rst +++ /dev/null @@ -1,3 +0,0 @@ -:class:`urllib.request.CacheFTPHandler` no longer raises :class:`URLError` -if a cached FTP instance is reused. ftplib's endtransfer method calls -voidresp to drain the connection to handle FTP instance reuse properly. diff --git a/Misc/NEWS.d/next/Library/2023-04-24-00-34-23.gh-issue-103685.U14jBM.rst b/Misc/NEWS.d/next/Library/2023-04-24-00-34-23.gh-issue-103685.U14jBM.rst deleted file mode 100644 index 31df04790721a8..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-24-00-34-23.gh-issue-103685.U14jBM.rst +++ /dev/null @@ -1 +0,0 @@ -Prepare :meth:`tkinter.Menu.index` for Tk 8.7 so that it does not raise ``TclError: expected integer but got ""`` when it should return ``None``. diff --git a/Misc/NEWS.d/next/Library/2023-04-24-16-00-28.gh-issue-90750.da0Xi8.rst b/Misc/NEWS.d/next/Library/2023-04-24-16-00-28.gh-issue-90750.da0Xi8.rst deleted file mode 100644 index 99e10f140f5049..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-24-16-00-28.gh-issue-90750.da0Xi8.rst +++ /dev/null @@ -1,3 +0,0 @@ -Use :meth:`datetime.datetime.fromisocalendar` in the implementation of -:meth:`datetime.datetime.strptime`, which should now accept only valid ISO -dates. (Patch by Paul Ganssle) diff --git a/Misc/NEWS.d/next/Library/2023-04-24-23-07-56.gh-issue-103791.bBPWdS.rst b/Misc/NEWS.d/next/Library/2023-04-24-23-07-56.gh-issue-103791.bBPWdS.rst deleted file mode 100644 index f00384cde9706e..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-24-23-07-56.gh-issue-103791.bBPWdS.rst +++ /dev/null @@ -1,3 +0,0 @@ -:class:`contextlib.suppress` now supports suppressing exceptions raised as -part of an :exc:`ExceptionGroup`. If other exceptions exist on the group, they -are re-raised in a group that does not contain the suppressed exceptions. diff --git a/Misc/NEWS.d/next/Library/2023-04-25-16-31-00.gh-issue-103839.tpyLhI.rst b/Misc/NEWS.d/next/Library/2023-04-25-16-31-00.gh-issue-103839.tpyLhI.rst deleted file mode 100644 index 1d40a8fbcd3f59..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-25-16-31-00.gh-issue-103839.tpyLhI.rst +++ /dev/null @@ -1,2 +0,0 @@ -Avoid compilation error due to tommath.h not being found when building -Tkinter against Tcl 8.7 built with bundled libtommath. diff --git a/Misc/NEWS.d/next/Library/2023-04-25-17-03-18.gh-issue-103857.Mr2Cak.rst b/Misc/NEWS.d/next/Library/2023-04-25-17-03-18.gh-issue-103857.Mr2Cak.rst deleted file mode 100644 index 3bd370dabf4ed5..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-25-17-03-18.gh-issue-103857.Mr2Cak.rst +++ /dev/null @@ -1,2 +0,0 @@ -Deprecated :meth:`datetime.datetime.utcnow` and -:meth:`datetime.datetime.utcfromtimestamp`. (Patch by Paul Ganssle) diff --git a/Misc/NEWS.d/next/Library/2023-04-25-19-58-13.gh-issue-103861.JeozgD.rst b/Misc/NEWS.d/next/Library/2023-04-25-19-58-13.gh-issue-103861.JeozgD.rst deleted file mode 100644 index cc1c444449fe93..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-25-19-58-13.gh-issue-103861.JeozgD.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix ``zipfile.Zipfile`` creating invalid zip files when ``force_zip64`` was -used to add files to them. Patch by Carey Metcalfe. diff --git a/Misc/NEWS.d/next/Library/2023-04-25-22-06-00.gh-issue-74940.TOacQ9.rst b/Misc/NEWS.d/next/Library/2023-04-25-22-06-00.gh-issue-74940.TOacQ9.rst deleted file mode 100644 index c37d795f3eb33d..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-25-22-06-00.gh-issue-74940.TOacQ9.rst +++ /dev/null @@ -1,2 +0,0 @@ -The C.UTF-8 locale is no longer converted to en_US.UTF-8, enabling the use -of UTF-8 encoding on systems which have no locales installed. diff --git a/Misc/NEWS.d/next/Library/2023-04-25-22-59-06.gh-issue-99944.pst8iT.rst b/Misc/NEWS.d/next/Library/2023-04-25-22-59-06.gh-issue-99944.pst8iT.rst deleted file mode 100644 index 80238a65e32a41..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-25-22-59-06.gh-issue-99944.pst8iT.rst +++ /dev/null @@ -1 +0,0 @@ -Make :mod:`dis` display the value of oparg of :opcode:`KW_NAMES`. diff --git a/Misc/NEWS.d/next/Library/2023-04-26-09-38-47.gh-issue-103872.8LBsDz.rst b/Misc/NEWS.d/next/Library/2023-04-26-09-38-47.gh-issue-103872.8LBsDz.rst deleted file mode 100644 index b840f9f5769f08..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-26-09-38-47.gh-issue-103872.8LBsDz.rst +++ /dev/null @@ -1 +0,0 @@ -Update the bundled copy of pip to version 23.1.2. diff --git a/Misc/NEWS.d/next/Library/2023-04-26-09-54-25.gh-issue-103848.aDSnpR.rst b/Misc/NEWS.d/next/Library/2023-04-26-09-54-25.gh-issue-103848.aDSnpR.rst deleted file mode 100644 index 81e5904aa6cca2..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-26-09-54-25.gh-issue-103848.aDSnpR.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add checks to ensure that ``[`` bracketed ``]`` hosts found by -:func:`urllib.parse.urlsplit` are of IPv6 or IPvFuture format. diff --git a/Misc/NEWS.d/next/Library/2023-04-26-15-14-36.gh-issue-103583.iCMDFt.rst b/Misc/NEWS.d/next/Library/2023-04-26-15-14-36.gh-issue-103583.iCMDFt.rst deleted file mode 100644 index 8c92ee40831619..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-26-15-14-36.gh-issue-103583.iCMDFt.rst +++ /dev/null @@ -1,2 +0,0 @@ -Isolate :mod:`!_multibytecodec` and codecs extension modules. Patches by -Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2023-04-26-18-12-13.gh-issue-103636.-KvCgO.rst b/Misc/NEWS.d/next/Library/2023-04-26-18-12-13.gh-issue-103636.-KvCgO.rst deleted file mode 100644 index a05a6f5cbcdb99..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-26-18-12-13.gh-issue-103636.-KvCgO.rst +++ /dev/null @@ -1 +0,0 @@ -Module-level attributes ``January`` and ``February`` are deprecated from :mod:`calendar`. diff --git a/Misc/NEWS.d/next/Library/2023-04-27-00-05-32.gh-issue-102628.X230E-.rst b/Misc/NEWS.d/next/Library/2023-04-27-00-05-32.gh-issue-102628.X230E-.rst deleted file mode 100644 index eaaca5b41ba5e2..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-27-00-05-32.gh-issue-102628.X230E-.rst +++ /dev/null @@ -1,2 +0,0 @@ -Substitute CTRL-D with CTRL-Z in :mod:`sqlite3` CLI banner when running on -Windows. diff --git a/Misc/NEWS.d/next/Library/2023-04-27-00-45-41.gh-issue-100370.MgZ3KY.rst b/Misc/NEWS.d/next/Library/2023-04-27-00-45-41.gh-issue-100370.MgZ3KY.rst deleted file mode 100644 index 9022d55c48cb11..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-27-00-45-41.gh-issue-100370.MgZ3KY.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix potential :exc:`OverflowError` in :meth:`sqlite3.Connection.blobopen` -for 32-bit builds. Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2023-04-27-18-46-31.gh-issue-68968.E3tnhy.rst b/Misc/NEWS.d/next/Library/2023-04-27-18-46-31.gh-issue-68968.E3tnhy.rst deleted file mode 100644 index bf29b64793b933..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-27-18-46-31.gh-issue-68968.E3tnhy.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed garbled output of :meth:`~unittest.TestCase.assertEqual` when an input lacks final newline. diff --git a/Misc/NEWS.d/next/Library/2023-04-27-20-03-08.gh-issue-103935.Uaf2M0.rst b/Misc/NEWS.d/next/Library/2023-04-27-20-03-08.gh-issue-103935.Uaf2M0.rst deleted file mode 100644 index 71b2d87249c47b..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-27-20-03-08.gh-issue-103935.Uaf2M0.rst +++ /dev/null @@ -1 +0,0 @@ -Use :func:`io.open_code` for files to be executed instead of raw :func:`open` diff --git a/Misc/NEWS.d/next/Library/2023-04-28-18-04-23.gh-issue-88773.xXCNJw.rst b/Misc/NEWS.d/next/Library/2023-04-28-18-04-23.gh-issue-88773.xXCNJw.rst deleted file mode 100644 index f14c9533f3af87..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-28-18-04-23.gh-issue-88773.xXCNJw.rst +++ /dev/null @@ -1 +0,0 @@ -Added :func:`turtle.teleport` to the :mod:`turtle` module to move a turtle to a new point without tracing a line, visible or invisible. Patch by Liam Gersten. diff --git a/Misc/NEWS.d/next/Library/2023-04-28-19-08-50.gh-issue-103977.msF70A.rst b/Misc/NEWS.d/next/Library/2023-04-28-19-08-50.gh-issue-103977.msF70A.rst deleted file mode 100644 index ff4005774a95d2..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-28-19-08-50.gh-issue-103977.msF70A.rst +++ /dev/null @@ -1 +0,0 @@ -Improve import time of :mod:`platform` module. diff --git a/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst b/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst deleted file mode 100644 index 48c6d149a8ed42..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst +++ /dev/null @@ -1,2 +0,0 @@ -In :mod:`mmap`, fix several bugs that could lead to access to memory-mapped files after -they have been invalidated. diff --git a/Misc/NEWS.d/next/Library/2023-05-01-16-43-28.gh-issue-104035.MrJBw8.rst b/Misc/NEWS.d/next/Library/2023-05-01-16-43-28.gh-issue-104035.MrJBw8.rst deleted file mode 100644 index 8c8e3d6ba5fbc1..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-01-16-43-28.gh-issue-104035.MrJBw8.rst +++ /dev/null @@ -1,2 +0,0 @@ -Do not ignore user-defined ``__getstate__`` and ``__setstate__`` methods for -slotted frozen dataclasses. diff --git a/Misc/NEWS.d/next/Library/2023-05-01-17-58-28.gh-issue-103963.XWlHx7.rst b/Misc/NEWS.d/next/Library/2023-05-01-17-58-28.gh-issue-103963.XWlHx7.rst deleted file mode 100644 index cb06ad5d22e8a9..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-01-17-58-28.gh-issue-103963.XWlHx7.rst +++ /dev/null @@ -1 +0,0 @@ -Make :mod:`dis` display the names of the args for :opcode:`CALL_INTRINSIC_*`. diff --git a/Misc/NEWS.d/next/Library/2023-05-01-19-10-05.gh-issue-103629.81bpZz.rst b/Misc/NEWS.d/next/Library/2023-05-01-19-10-05.gh-issue-103629.81bpZz.rst deleted file mode 100644 index 7971ab66359c3d..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-01-19-10-05.gh-issue-103629.81bpZz.rst +++ /dev/null @@ -1 +0,0 @@ -Update the ``repr`` of :class:`typing.Unpack` according to :pep:`692`. diff --git a/Misc/NEWS.d/next/Library/2023-05-02-04-49-45.gh-issue-103822.m0QdAO.rst b/Misc/NEWS.d/next/Library/2023-05-02-04-49-45.gh-issue-103822.m0QdAO.rst deleted file mode 100644 index 3daf9cc093807b..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-02-04-49-45.gh-issue-103822.m0QdAO.rst +++ /dev/null @@ -1 +0,0 @@ -Update the return type of ``weekday`` to the newly added Day attribute diff --git a/Misc/NEWS.d/next/Library/2023-05-02-20-43-03.gh-issue-104102.vgSdEJ.rst b/Misc/NEWS.d/next/Library/2023-05-02-20-43-03.gh-issue-104102.vgSdEJ.rst deleted file mode 100644 index 7101de908a5004..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-02-20-43-03.gh-issue-104102.vgSdEJ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve performance of :meth:`pathlib.Path.glob` when evaluating patterns -that contain ``'../'`` segments. diff --git a/Misc/NEWS.d/next/Library/2023-05-02-21-05-30.gh-issue-104104.9tjplT.rst b/Misc/NEWS.d/next/Library/2023-05-02-21-05-30.gh-issue-104104.9tjplT.rst deleted file mode 100644 index 935a0e2a2bff18..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-02-21-05-30.gh-issue-104104.9tjplT.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve performance of :meth:`pathlib.Path.glob` by using -:data:`re.IGNORECASE` to implement case-insensitive matching. diff --git a/Misc/NEWS.d/next/Library/2023-05-03-03-14-33.gh-issue-104114.RG26RD.rst b/Misc/NEWS.d/next/Library/2023-05-03-03-14-33.gh-issue-104114.RG26RD.rst deleted file mode 100644 index e705fea8326e7a..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-03-03-14-33.gh-issue-104114.RG26RD.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix issue where :meth:`pathlib.Path.glob` returns paths using the case of -non-wildcard segments for corresponding path segments, rather than the real -filesystem case. diff --git a/Misc/NEWS.d/next/Library/2023-05-03-16-50-24.gh-issue-104144.yNkjL8.rst b/Misc/NEWS.d/next/Library/2023-05-03-16-50-24.gh-issue-104144.yNkjL8.rst deleted file mode 100644 index b975d48ed3385c..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-03-16-50-24.gh-issue-104144.yNkjL8.rst +++ /dev/null @@ -1,3 +0,0 @@ -Optimize :func:`asyncio.gather` when using :func:`asyncio.eager_task_factory` -to complete eagerly if all fututres completed eagerly. -Avoid scheduling done callbacks for futures that complete eagerly. diff --git a/Misc/NEWS.d/next/Library/2023-05-03-16-51-53.gh-issue-104144.653Q0P.rst b/Misc/NEWS.d/next/Library/2023-05-03-16-51-53.gh-issue-104144.653Q0P.rst deleted file mode 100644 index ced3b7cea04954..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-03-16-51-53.gh-issue-104144.653Q0P.rst +++ /dev/null @@ -1,2 +0,0 @@ -Optimize :class:`asyncio.TaskGroup` when using :func:`asyncio.eager_task_factory`. -Skip scheduling a done callback if a TaskGroup task completes eagerly. diff --git a/Misc/NEWS.d/next/Library/2023-05-03-19-22-24.gh-issue-90208.tI00da.rst b/Misc/NEWS.d/next/Library/2023-05-03-19-22-24.gh-issue-90208.tI00da.rst deleted file mode 100644 index 1fd9588bebd5d8..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-03-19-22-24.gh-issue-90208.tI00da.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fixed issue where :meth:`pathlib.Path.glob` returned incomplete results when -it encountered a :exc:`PermissionError`. This method now suppresses all -:exc:`OSError` exceptions, except those raised from calling -:meth:`~pathlib.Path.is_dir` on the top-level path. diff --git a/Misc/NEWS.d/next/Library/2023-05-05-18-52-22.gh-issue-65772.w5P5Wv.rst b/Misc/NEWS.d/next/Library/2023-05-05-18-52-22.gh-issue-65772.w5P5Wv.rst deleted file mode 100644 index 54b0190192863c..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-05-18-52-22.gh-issue-65772.w5P5Wv.rst +++ /dev/null @@ -1 +0,0 @@ -Remove unneeded comments and code in turtle.py. diff --git a/Misc/NEWS.d/next/Library/2023-05-06-20-37-46.gh-issue-102613.QZG9iX.rst b/Misc/NEWS.d/next/Library/2023-05-06-20-37-46.gh-issue-102613.QZG9iX.rst deleted file mode 100644 index 01f8b948d2cb65..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-06-20-37-46.gh-issue-102613.QZG9iX.rst +++ /dev/null @@ -1,3 +0,0 @@ -Improve performance of :meth:`pathlib.Path.glob` when expanding recursive -wildcards ("``**``") by merging adjacent wildcards and de-duplicating -results only when necessary. diff --git a/Misc/NEWS.d/next/Library/2023-05-07-19-56-45.gh-issue-104265.fVblry.rst b/Misc/NEWS.d/next/Library/2023-05-07-19-56-45.gh-issue-104265.fVblry.rst deleted file mode 100644 index 9c582844bf909b..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-07-19-56-45.gh-issue-104265.fVblry.rst +++ /dev/null @@ -1,4 +0,0 @@ -Prevent possible crash by disallowing instantiation of the -:class:`!_csv.Reader` and :class:`!_csv.Writer` types. -The regression was introduced in 3.10.0a4 with PR 23224 (:issue:`14935`). -Patch by Radislav Chugunov. diff --git a/Misc/NEWS.d/next/Library/2023-05-08-15-39-00.gh-issue-87695.f6iO7v.rst b/Misc/NEWS.d/next/Library/2023-05-08-15-39-00.gh-issue-87695.f6iO7v.rst deleted file mode 100644 index 7b677656bea308..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-08-15-39-00.gh-issue-87695.f6iO7v.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix issue where :meth:`pathlib.Path.glob` raised :exc:`OSError` when it -encountered a symlink to an overly long path. diff --git a/Misc/NEWS.d/next/Library/2023-05-08-15-50-59.gh-issue-104310.fXVSPY.rst b/Misc/NEWS.d/next/Library/2023-05-08-15-50-59.gh-issue-104310.fXVSPY.rst deleted file mode 100644 index 3743d569995f2e..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-08-15-50-59.gh-issue-104310.fXVSPY.rst +++ /dev/null @@ -1,3 +0,0 @@ -Users may now use ``importlib.util.allowing_all_extensions()`` (a context -manager) to temporarily disable the strict compatibility checks for -importing extension modules in subinterpreters. diff --git a/Misc/NEWS.d/next/Library/2023-05-08-20-57-17.gh-issue-104307.DSB93G.rst b/Misc/NEWS.d/next/Library/2023-05-08-20-57-17.gh-issue-104307.DSB93G.rst deleted file mode 100644 index 03775845450caa..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-08-20-57-17.gh-issue-104307.DSB93G.rst +++ /dev/null @@ -1 +0,0 @@ -:func:`socket.getnameinfo` now releases the GIL while contacting the DNS server diff --git a/Misc/NEWS.d/next/Library/2023-05-08-23-01-59.gh-issue-104139.83Tnt-.rst b/Misc/NEWS.d/next/Library/2023-05-08-23-01-59.gh-issue-104139.83Tnt-.rst deleted file mode 100644 index 145e75f6dea6f4..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-08-23-01-59.gh-issue-104139.83Tnt-.rst +++ /dev/null @@ -1,3 +0,0 @@ -Teach :func:`urllib.parse.unsplit` to retain the ``"//"`` when assembling -``itms-services://?action=generate-bugs`` style `Apple Platform Deployment -`_ URLs. diff --git a/Misc/NEWS.d/next/Library/2023-05-09-18-46-24.gh-issue-104301.gNnbId.rst b/Misc/NEWS.d/next/Library/2023-05-09-18-46-24.gh-issue-104301.gNnbId.rst deleted file mode 100644 index a44ad765e62458..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-09-18-46-24.gh-issue-104301.gNnbId.rst +++ /dev/null @@ -1 +0,0 @@ -Allow leading whitespace in disambiguated statements in :mod:`pdb`. diff --git a/Misc/NEWS.d/next/Library/2023-05-10-19-33-36.gh-issue-103000.j0KSfD.rst b/Misc/NEWS.d/next/Library/2023-05-10-19-33-36.gh-issue-103000.j0KSfD.rst deleted file mode 100644 index f84ec5c8b3caf6..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-10-19-33-36.gh-issue-103000.j0KSfD.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve performance of :func:`dataclasses.asdict` for the common case where -*dict_factory* is ``dict``. Patch by David C Ellis. diff --git a/Misc/NEWS.d/next/Library/2023-05-11-01-07-42.gh-issue-102613.uMsokt.rst b/Misc/NEWS.d/next/Library/2023-05-11-01-07-42.gh-issue-102613.uMsokt.rst deleted file mode 100644 index 3b06964dc8d233..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-11-01-07-42.gh-issue-102613.uMsokt.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix issue where :meth:`pathlib.Path.glob` raised :exc:`RecursionError` when -walking deep directory trees. diff --git a/Misc/NEWS.d/next/Library/2023-05-11-07-50-00.gh-issue-104392.YSllzt.rst b/Misc/NEWS.d/next/Library/2023-05-11-07-50-00.gh-issue-104392.YSllzt.rst deleted file mode 100644 index b441b2acd19f1b..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-11-07-50-00.gh-issue-104392.YSllzt.rst +++ /dev/null @@ -1,2 +0,0 @@ -Remove undocumented and unused ``_paramspec_tvars`` attribute from some -classes in :mod:`typing`. diff --git a/Misc/NEWS.d/next/Library/2023-05-11-21-32-18.gh-issue-101520.l9MjRE.rst b/Misc/NEWS.d/next/Library/2023-05-11-21-32-18.gh-issue-101520.l9MjRE.rst deleted file mode 100644 index 5e8bf967bfdcca..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-11-21-32-18.gh-issue-101520.l9MjRE.rst +++ /dev/null @@ -1,2 +0,0 @@ -Move the core functionality of the ``tracemalloc`` module in the ``Python/`` -folder, leaving just the module wrapper in ``Modules/``. diff --git a/Misc/NEWS.d/next/Library/2023-05-12-19-29-28.gh-issue-103857.0IzSxr.rst b/Misc/NEWS.d/next/Library/2023-05-12-19-29-28.gh-issue-103857.0IzSxr.rst deleted file mode 100644 index 6e8162d9ecc286..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-12-19-29-28.gh-issue-103857.0IzSxr.rst +++ /dev/null @@ -1 +0,0 @@ -Update datetime deprecations' stracktrace to point to the calling line diff --git a/Misc/NEWS.d/next/Library/2023-05-16-10-07-16.gh-issue-104536.hFWD8f.rst b/Misc/NEWS.d/next/Library/2023-05-16-10-07-16.gh-issue-104536.hFWD8f.rst deleted file mode 100644 index b0f5d78f7e61da..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-16-10-07-16.gh-issue-104536.hFWD8f.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a race condition in the internal :mod:`multiprocessing.process` cleanup -logic that could manifest as an unintended ``AttributeError`` when calling -``process.close()``. diff --git a/Misc/NEWS.d/next/Library/2023-05-16-11-02-44.gh-issue-75367.qLWR35.rst b/Misc/NEWS.d/next/Library/2023-05-16-11-02-44.gh-issue-75367.qLWR35.rst deleted file mode 100644 index 554c425e6a7864..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-16-11-02-44.gh-issue-75367.qLWR35.rst +++ /dev/null @@ -1 +0,0 @@ -Fix data descriptor detection in :func:`inspect.getattr_static`. diff --git a/Misc/NEWS.d/next/Library/2023-05-17-03-14-07.gh-issue-104484.y6KxL6.rst b/Misc/NEWS.d/next/Library/2023-05-17-03-14-07.gh-issue-104484.y6KxL6.rst deleted file mode 100644 index 6d42078c35dd26..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-17-03-14-07.gh-issue-104484.y6KxL6.rst +++ /dev/null @@ -1 +0,0 @@ -Added *case_sensitive* argument to :meth:`pathlib.PurePath.match` diff --git a/Misc/NEWS.d/next/Library/2023-05-17-08-01-36.gh-issue-104372.jpoWs6.rst b/Misc/NEWS.d/next/Library/2023-05-17-08-01-36.gh-issue-104372.jpoWs6.rst deleted file mode 100644 index c228f503aab4bd..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-17-08-01-36.gh-issue-104372.jpoWs6.rst +++ /dev/null @@ -1 +0,0 @@ -Refactored the ``_posixsubprocess`` internals to avoid Python C API usage between fork and exec when marking ``pass_fds=`` file descriptors inheritable. diff --git a/Misc/NEWS.d/next/Library/2023-05-17-16-58-23.gh-issue-104555.5rb5oM.rst b/Misc/NEWS.d/next/Library/2023-05-17-16-58-23.gh-issue-104555.5rb5oM.rst deleted file mode 100644 index 2992346484c5fa..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-17-16-58-23.gh-issue-104555.5rb5oM.rst +++ /dev/null @@ -1,7 +0,0 @@ -Fix issue where an :func:`issubclass` check comparing a class ``X`` against a -:func:`runtime-checkable protocol ` ``Y`` with -non-callable members would not cause :exc:`TypeError` to be raised if an -:func:`isinstance` call had previously been made comparing an instance of ``X`` -to ``Y``. This issue was present in edge cases on Python 3.11, but became more -prominent in 3.12 due to some unrelated changes that were made to -runtime-checkable protocols. Patch by Alex Waygood. diff --git a/Misc/NEWS.d/next/Library/2023-05-17-20-03-01.gh-issue-104340.kp_XmX.rst b/Misc/NEWS.d/next/Library/2023-05-17-20-03-01.gh-issue-104340.kp_XmX.rst deleted file mode 100644 index 5b03622df6a2ab..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-17-20-03-01.gh-issue-104340.kp_XmX.rst +++ /dev/null @@ -1 +0,0 @@ -When an ``asyncio`` pipe protocol loses its connection due to an error, and the caller doesn't await ``wait_closed()`` on the corresponding ``StreamWriter``, don't log a warning about an exception that was never retrieved. After all, according to the ``StreamWriter.close()`` docs, the ``wait_closed()`` call is optional ("not mandatory"). diff --git a/Misc/NEWS.d/next/Library/2023-05-17-21-01-48.gh-issue-104600.E6CK35.rst b/Misc/NEWS.d/next/Library/2023-05-17-21-01-48.gh-issue-104600.E6CK35.rst deleted file mode 100644 index 64f81e140d9639..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-17-21-01-48.gh-issue-104600.E6CK35.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`functools.update_wrapper` now sets the ``__type_params__`` attribute -(added by :pep:`695`). diff --git a/Misc/NEWS.d/next/Security/2023-03-07-20-59-17.gh-issue-102153.14CLSZ.rst b/Misc/NEWS.d/next/Security/2023-03-07-20-59-17.gh-issue-102153.14CLSZ.rst deleted file mode 100644 index e57ac4ed3ac5d7..00000000000000 --- a/Misc/NEWS.d/next/Security/2023-03-07-20-59-17.gh-issue-102153.14CLSZ.rst +++ /dev/null @@ -1,3 +0,0 @@ -:func:`urllib.parse.urlsplit` now strips leading C0 control and space -characters following the specification for URLs defined by WHATWG in -response to CVE-2023-24329. Patch by Illia Volochii. diff --git a/Misc/NEWS.d/next/Security/2023-04-17-14-38-12.gh-issue-99108.720lG8.rst b/Misc/NEWS.d/next/Security/2023-04-17-14-38-12.gh-issue-99108.720lG8.rst deleted file mode 100644 index f259acf753831c..00000000000000 --- a/Misc/NEWS.d/next/Security/2023-04-17-14-38-12.gh-issue-99108.720lG8.rst +++ /dev/null @@ -1,2 +0,0 @@ -Upgrade built-in :mod:`hashlib` SHA3 implementation to a verified implementation -from the ``HACL*`` project. Used when OpenSSL is not present or lacks SHA3. diff --git a/Misc/NEWS.d/next/Security/2023-05-01-15-03-25.gh-issue-104049.b01Y3g.rst b/Misc/NEWS.d/next/Security/2023-05-01-15-03-25.gh-issue-104049.b01Y3g.rst deleted file mode 100644 index 969deb26bfeb95..00000000000000 --- a/Misc/NEWS.d/next/Security/2023-05-01-15-03-25.gh-issue-104049.b01Y3g.rst +++ /dev/null @@ -1,2 +0,0 @@ -Do not expose the local on-disk location in directory indexes -produced by :class:`http.client.SimpleHTTPRequestHandler`. diff --git a/Misc/NEWS.d/next/Security/2023-05-02-17-56-32.gh-issue-99889.l664SU.rst b/Misc/NEWS.d/next/Security/2023-05-02-17-56-32.gh-issue-99889.l664SU.rst deleted file mode 100644 index b7002e81b6b677..00000000000000 --- a/Misc/NEWS.d/next/Security/2023-05-02-17-56-32.gh-issue-99889.l664SU.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed a security in flaw in :func:`uu.decode` that could allow for -directory traversal based on the input if no ``out_file`` was specified. diff --git a/Misc/NEWS.d/next/Tests/2022-11-06-18-42-38.gh-issue-75729.uGYJrv.rst b/Misc/NEWS.d/next/Tests/2022-11-06-18-42-38.gh-issue-75729.uGYJrv.rst deleted file mode 100644 index 8baecdfc31881f..00000000000000 --- a/Misc/NEWS.d/next/Tests/2022-11-06-18-42-38.gh-issue-75729.uGYJrv.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix the :func:`os.spawn* ` tests failing on Windows -when the working directory or interpreter path contains spaces. diff --git a/Misc/NEWS.d/next/Tests/2023-03-17-22-00-47.gh-issue-102795.z21EoC.rst b/Misc/NEWS.d/next/Tests/2023-03-17-22-00-47.gh-issue-102795.z21EoC.rst deleted file mode 100644 index fe2afff91ece7a..00000000000000 --- a/Misc/NEWS.d/next/Tests/2023-03-17-22-00-47.gh-issue-102795.z21EoC.rst +++ /dev/null @@ -1 +0,0 @@ -fix use of poll in test_epoll's test_control_and_wait diff --git a/Misc/NEWS.d/next/Tests/2023-04-08-00-50-23.gh-issue-103329.M38tqF.rst b/Misc/NEWS.d/next/Tests/2023-04-08-00-50-23.gh-issue-103329.M38tqF.rst deleted file mode 100644 index 79448ed728040d..00000000000000 --- a/Misc/NEWS.d/next/Tests/2023-04-08-00-50-23.gh-issue-103329.M38tqF.rst +++ /dev/null @@ -1 +0,0 @@ -Regression tests for the behaviour of ``unittest.mock.PropertyMock`` were added. diff --git a/Misc/NEWS.d/next/Tests/2023-04-25-12-19-37.gh-issue-86275.-RoLIt.rst b/Misc/NEWS.d/next/Tests/2023-04-25-12-19-37.gh-issue-86275.-RoLIt.rst deleted file mode 100644 index 37ab74ffc441ed..00000000000000 --- a/Misc/NEWS.d/next/Tests/2023-04-25-12-19-37.gh-issue-86275.-RoLIt.rst +++ /dev/null @@ -1,2 +0,0 @@ -Added property-based tests to the :mod:`zoneinfo` tests, along with stubs -for the ``hypothesis`` interface. (Patch by Paul Ganssle) diff --git a/Misc/NEWS.d/next/Tests/2023-05-14-03-00-00.gh-issue-104461.Rmex11.rst b/Misc/NEWS.d/next/Tests/2023-05-14-03-00-00.gh-issue-104461.Rmex11.rst deleted file mode 100644 index ae69f623e94558..00000000000000 --- a/Misc/NEWS.d/next/Tests/2023-05-14-03-00-00.gh-issue-104461.Rmex11.rst +++ /dev/null @@ -1,3 +0,0 @@ -Run test_configure_screen on X11 only, since the ``DISPLAY`` -environment variable and ``-screen`` option for toplevels -are not useful on Tk for Win32 or Aqua. diff --git a/Misc/NEWS.d/next/Tests/2023-05-15-02-22-44.gh-issue-104494.Bkrbfn.rst b/Misc/NEWS.d/next/Tests/2023-05-15-02-22-44.gh-issue-104494.Bkrbfn.rst deleted file mode 100644 index a320c48428b58b..00000000000000 --- a/Misc/NEWS.d/next/Tests/2023-05-15-02-22-44.gh-issue-104494.Bkrbfn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update ``test_pack_configure_in`` and ``test_place_configure_in`` -for changes to error message formatting in Tk 8.7. diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-05-11-15-12-11.gh-issue-104389.EiOhB3.rst b/Misc/NEWS.d/next/Tools-Demos/2023-05-11-15-12-11.gh-issue-104389.EiOhB3.rst deleted file mode 100644 index 854e1cca967c42..00000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2023-05-11-15-12-11.gh-issue-104389.EiOhB3.rst +++ /dev/null @@ -1,2 +0,0 @@ -Argument Clinic C converters now accept the ``unused`` keyword, for wrapping -a parameter with :c:macro:`Py_UNUSED`. Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Windows/2023-03-18-21-38-00.gh-issue-88013.Z3loxC.rst b/Misc/NEWS.d/next/Windows/2023-03-18-21-38-00.gh-issue-88013.Z3loxC.rst deleted file mode 100644 index 4ca3185ea1f65e..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-03-18-21-38-00.gh-issue-88013.Z3loxC.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed a bug where :exc:`TypeError` was raised when calling -:func:`ntpath.realpath` with a bytes parameter in some cases. diff --git a/Misc/NEWS.d/next/Windows/2023-03-24-11-25-28.gh-issue-102997.dredy2.rst b/Misc/NEWS.d/next/Windows/2023-03-24-11-25-28.gh-issue-102997.dredy2.rst deleted file mode 100644 index c8f7259aecba6f..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-03-24-11-25-28.gh-issue-102997.dredy2.rst +++ /dev/null @@ -1 +0,0 @@ -Update Windows installer to use SQLite 3.41.2. diff --git a/Misc/NEWS.d/next/Windows/2023-04-11-09-22-22.gh-issue-103088.6AJEuR.rst b/Misc/NEWS.d/next/Windows/2023-04-11-09-22-22.gh-issue-103088.6AJEuR.rst deleted file mode 100644 index f9f5343f4210dc..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-04-11-09-22-22.gh-issue-103088.6AJEuR.rst +++ /dev/null @@ -1 +0,0 @@ -Fixes venvs not working in bash on Windows across different disks diff --git a/Misc/NEWS.d/next/Windows/2023-04-12-10-49-21.gh-issue-103088.Yjj-qJ.rst b/Misc/NEWS.d/next/Windows/2023-04-12-10-49-21.gh-issue-103088.Yjj-qJ.rst deleted file mode 100644 index 1fee99da240378..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-04-12-10-49-21.gh-issue-103088.Yjj-qJ.rst +++ /dev/null @@ -1 +0,0 @@ -Fix virtual environment :file:`activate` script having incorrect line endings for Cygwin. diff --git a/Misc/NEWS.d/next/Windows/2023-04-24-15-51-11.gh-issue-82814.GI3UkZ.rst b/Misc/NEWS.d/next/Windows/2023-04-24-15-51-11.gh-issue-82814.GI3UkZ.rst deleted file mode 100644 index 5bd005ffacb800..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-04-24-15-51-11.gh-issue-82814.GI3UkZ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a potential ``[Errno 13] Permission denied`` when using :func:`shutil.copystat` -within Windows Subsystem for Linux (WSL) on a mounted filesystem by adding -``errno.EACCES`` to the list of ignored errors within the internal implementation. diff --git a/Misc/NEWS.d/next/Windows/2023-05-18-22-46-03.gh-issue-104623.HJZhm1.rst b/Misc/NEWS.d/next/Windows/2023-05-18-22-46-03.gh-issue-104623.HJZhm1.rst deleted file mode 100644 index 3ffe8261e5a1b0..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-05-18-22-46-03.gh-issue-104623.HJZhm1.rst +++ /dev/null @@ -1 +0,0 @@ -Update Windows installer to use SQLite 3.42.0. diff --git a/Misc/NEWS.d/next/macOS/2023-03-24-11-20-47.gh-issue-102997.ZgQkbq.rst b/Misc/NEWS.d/next/macOS/2023-03-24-11-20-47.gh-issue-102997.ZgQkbq.rst deleted file mode 100644 index d0b390a896b743..00000000000000 --- a/Misc/NEWS.d/next/macOS/2023-03-24-11-20-47.gh-issue-102997.ZgQkbq.rst +++ /dev/null @@ -1 +0,0 @@ -Update macOS installer to SQLite 3.41.2. diff --git a/Misc/NEWS.d/next/macOS/2023-04-04-13-37-28.gh-issue-103207.x0vvQp.rst b/Misc/NEWS.d/next/macOS/2023-04-04-13-37-28.gh-issue-103207.x0vvQp.rst deleted file mode 100644 index 3c176e3a6b5310..00000000000000 --- a/Misc/NEWS.d/next/macOS/2023-04-04-13-37-28.gh-issue-103207.x0vvQp.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add instructions to the macOS installer welcome display on how to workaround -the macOS 13 Ventura “The installer encountered an error” failure. diff --git a/Misc/NEWS.d/next/macOS/2023-04-24-18-37-48.gh-issue-60436.in-IyF.rst b/Misc/NEWS.d/next/macOS/2023-04-24-18-37-48.gh-issue-60436.in-IyF.rst deleted file mode 100644 index f274d3b898f15d..00000000000000 --- a/Misc/NEWS.d/next/macOS/2023-04-24-18-37-48.gh-issue-60436.in-IyF.rst +++ /dev/null @@ -1 +0,0 @@ -update curses textbox to additionally handle backspace using the ``curses.ascii.DEL`` key press. diff --git a/Misc/NEWS.d/next/macOS/2023-05-04-21-47-59.gh-issue-104180.lEJCwd.rst b/Misc/NEWS.d/next/macOS/2023-05-04-21-47-59.gh-issue-104180.lEJCwd.rst deleted file mode 100644 index b6b18dcfd81394..00000000000000 --- a/Misc/NEWS.d/next/macOS/2023-05-04-21-47-59.gh-issue-104180.lEJCwd.rst +++ /dev/null @@ -1,2 +0,0 @@ -Support reading SOCKS proxy configuration from macOS System Configuration. -Patch by Sam Schott. diff --git a/Misc/NEWS.d/next/macOS/2023-05-18-08-52-04.gh-issue-103545.pi5k2N.rst b/Misc/NEWS.d/next/macOS/2023-05-18-08-52-04.gh-issue-103545.pi5k2N.rst deleted file mode 100644 index c40f9460508dd7..00000000000000 --- a/Misc/NEWS.d/next/macOS/2023-05-18-08-52-04.gh-issue-103545.pi5k2N.rst +++ /dev/null @@ -1 +0,0 @@ -Add ``os.PRIO_DARWIN_THREAD``, ``os.PRIO_DARWIN_PROCESS``, ``os.PRIO_DARWIN_BG`` and ``os.PRIO_DARWIN_NONUI``. These can be used with ``os.setpriority`` to run the process at a lower priority and make use of the efficiency cores on Apple Silicon systems. diff --git a/Misc/NEWS.d/next/macOS/2023-05-18-22-31-49.gh-issue-104623.6h7Xfx.rst b/Misc/NEWS.d/next/macOS/2023-05-18-22-31-49.gh-issue-104623.6h7Xfx.rst deleted file mode 100644 index ffb124631c218b..00000000000000 --- a/Misc/NEWS.d/next/macOS/2023-05-18-22-31-49.gh-issue-104623.6h7Xfx.rst +++ /dev/null @@ -1 +0,0 @@ -Update macOS installer to SQLite 3.42.0. diff --git a/Misc/NEWS.d/next/macOS/2023-05-21-23-54-52.gh-issue-99834.6ANPts.rst b/Misc/NEWS.d/next/macOS/2023-05-21-23-54-52.gh-issue-99834.6ANPts.rst deleted file mode 100644 index aeb64d25b09add..00000000000000 --- a/Misc/NEWS.d/next/macOS/2023-05-21-23-54-52.gh-issue-99834.6ANPts.rst +++ /dev/null @@ -1 +0,0 @@ -Update macOS installer to Tcl/Tk 8.6.13. diff --git a/README.rst b/README.rst index 06c4f0a1fa3917..596a62108cf1f1 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.12.0 alpha 7 +This is Python version 3.12.0 beta 1 ===================================== .. image:: https://github.com/python/cpython/workflows/Tests/badge.svg From 151b6bfb5d9a15b6e2682e5a3008a3f9ec3086ae Mon Sep 17 00:00:00 2001 From: Mathieu Dupuy Date: Mon, 22 May 2023 15:03:29 +0200 Subject: [PATCH 22/47] howto/urllib2: remove link to an outdated french translation (GH-104193) We now have our own translation and it's not outdated --- Doc/howto/urllib2.rst | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index 61ba6bd7224fcc..86137fb38c9b93 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -6,13 +6,6 @@ :Author: `Michael Foord `_ -.. note:: - - There is a French translation of an earlier revision of this - HOWTO, available at `urllib2 - Le Manuel manquant - `_. - - Introduction ============ From e295d8605699ad3d8ec46c8d55a5e47da05b20c6 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 22 May 2023 15:12:19 +0100 Subject: [PATCH 23/47] GH-101291: Avoid using macros with casts in low-level long API. (#104742) --- Include/cpython/longintrepr.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index 0f569935fff14a..692c69ba76db2f 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -104,9 +104,10 @@ _PyLong_FromDigits(int negative, Py_ssize_t digit_count, digit *digits); #define _PyLong_SIGN_MASK 3 #define _PyLong_NON_SIZE_BITS 3 + static inline int _PyLong_IsCompact(const PyLongObject* op) { - assert(PyLong_Check(op)); + assert(PyType_HasFeature((op)->ob_base.ob_type, Py_TPFLAGS_LONG_SUBCLASS)); return op->long_value.lv_tag < (2 << _PyLong_NON_SIZE_BITS); } @@ -115,7 +116,7 @@ _PyLong_IsCompact(const PyLongObject* op) { static inline Py_ssize_t _PyLong_CompactValue(const PyLongObject *op) { - assert(PyLong_Check(op)); + assert(PyType_HasFeature((op)->ob_base.ob_type, Py_TPFLAGS_LONG_SUBCLASS)); assert(PyUnstable_Long_IsCompact(op)); Py_ssize_t sign = 1 - (op->long_value.lv_tag & _PyLong_SIGN_MASK); return sign * (Py_ssize_t)op->long_value.ob_digit[0]; From 500b0f4814e05b918f485ceebf9c7be9a52ad469 Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Mon, 22 May 2023 21:15:32 +0200 Subject: [PATCH 24/47] Python 3.13.0a0 --- Doc/tutorial/interpreter.rst | 10 +-- Doc/whatsnew/3.13 | 132 +++++++++++++++++++++++++++++++++++ Include/patchlevel.h | 8 +-- README.rst | 12 ++-- configure.ac | 4 +- 5 files changed, 149 insertions(+), 17 deletions(-) create mode 100644 Doc/whatsnew/3.13 diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index b71c61089e6dc1..42ebf2b3d294a8 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -10,13 +10,13 @@ Using the Python Interpreter Invoking the Interpreter ======================== -The Python interpreter is usually installed as :file:`/usr/local/bin/python3.12` +The Python interpreter is usually installed as :file:`/usr/local/bin/python3.13` on those machines where it is available; putting :file:`/usr/local/bin` in your Unix shell's search path makes it possible to start it by typing the command: .. code-block:: text - python3.12 + python3.13 to the shell. [#]_ Since the choice of the directory where the interpreter lives is an installation option, other places are possible; check with your local @@ -24,7 +24,7 @@ Python guru or system administrator. (E.g., :file:`/usr/local/python` is a popular alternative location.) On Windows machines where you have installed Python from the :ref:`Microsoft Store -`, the :file:`python3.12` command will be available. If you have +`, the :file:`python3.13` command will be available. If you have the :ref:`py.exe launcher ` installed, you can use the :file:`py` command. See :ref:`setting-envvars` for other ways to launch Python. @@ -97,8 +97,8 @@ before printing the first prompt: .. code-block:: shell-session - $ python3.12 - Python 3.12 (default, April 4 2022, 09:25:04) + $ python3.13 + Python 3.13 (default, April 4 2023, 09:25:04) [GCC 10.2.0] on linux Type "help", "copyright", "credits" or "license" for more information. >>> diff --git a/Doc/whatsnew/3.13 b/Doc/whatsnew/3.13 new file mode 100644 index 00000000000000..af331c46f1c0e4 --- /dev/null +++ b/Doc/whatsnew/3.13 @@ -0,0 +1,132 @@ + +**************************** + What's New In Python 3.13 +**************************** + +:Release: |release| +:Date: |today| + +.. Rules for maintenance: + + * Anyone can add text to this document. Do not spend very much time + on the wording of your changes, because your text will probably + get rewritten to some degree. + + * The maintainer will go through Misc/NEWS periodically and add + changes; it's therefore more important to add your changes to + Misc/NEWS than to this file. + + * This is not a complete list of every single change; completeness + is the purpose of Misc/NEWS. Some changes I consider too small + or esoteric to include. If such a change is added to the text, + I'll just remove it. (This is another reason you shouldn't spend + too much time on writing your addition.) + + * If you want to draw your new text to the attention of the + maintainer, add 'XXX' to the beginning of the paragraph or + section. + + * It's OK to just add a fragmentary note about a change. For + example: "XXX Describe the transmogrify() function added to the + socket module." The maintainer will research the change and + write the necessary text. + + * You can comment out your additions if you like, but it's not + necessary (especially when a final release is some months away). + + * Credit the author of a patch or bugfix. Just the name is + sufficient; the e-mail address isn't necessary. + + * It's helpful to add the issue number as a comment: + + XXX Describe the transmogrify() function added to the socket + module. + (Contributed by P.Y. Developer in :gh:`12345`.) + + This saves the maintainer the effort of going through the VCS log + when researching a change. + +This article explains the new features in Python 3.13, compared to 3.12. + +For full details, see the :ref:`changelog `. + +.. note:: + + Prerelease users should be aware that this document is currently in draft + form. It will be updated substantially as Python 3.13 moves towards release, + so it's worth checking back even after reading earlier versions. + + +Summary -- Release highlights +============================= + +.. This section singles out the most important changes in Python 3.13. + Brevity is key. + + +.. PEP-sized items next. + + + +New Features +============ + + + +Other Language Changes +====================== + + + +New Modules +=========== + +* None yet. + + +Improved Modules +================ + + +Optimizations +============= + + + + +Deprecated +========== + + + +Removed +======= + + + +Porting to Python 3.13 +====================== + +This section lists previously described changes and other bugfixes +that may require changes to your code. + + +Build Changes +============= + + +C API Changes +============= + +New Features +------------ + +Porting to Python 3.13 +---------------------- + +Deprecated +---------- + +Removed +------- + diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 3de9ed9296a47b..ae9d36c12d6eaf 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -17,13 +17,13 @@ /* Version parsed out into numeric values */ /*--start constants--*/ #define PY_MAJOR_VERSION 3 -#define PY_MINOR_VERSION 12 +#define PY_MINOR_VERSION 13 #define PY_MICRO_VERSION 0 -#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_BETA -#define PY_RELEASE_SERIAL 1 +#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA +#define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.12.0b1" +#define PY_VERSION "3.13.0a0" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/README.rst b/README.rst index 596a62108cf1f1..4a4c8ab65dceff 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.12.0 beta 1 +This is Python version 3.13.0 alpha 0 ===================================== .. image:: https://github.com/python/cpython/workflows/Tests/badge.svg @@ -133,7 +133,7 @@ What's New ---------- We have a comprehensive overview of the changes in the `What's New in Python -3.12 `_ document. For a more +3.13 `_ document. For a more detailed change log, read `Misc/NEWS `_, but a full accounting of changes can only be gleaned from the `commit history @@ -146,7 +146,7 @@ entitled "Installing multiple versions". Documentation ------------- -`Documentation for Python 3.12 `_ is online, +`Documentation for Python 3.13 `_ is online, updated daily. It can also be downloaded in many formats for faster access. The documentation @@ -206,8 +206,8 @@ intend to install multiple versions using the same prefix you must decide which version (if any) is your "primary" version. Install that version using ``make install``. Install all other versions using ``make altinstall``. -For example, if you want to install Python 2.7, 3.6, and 3.12 with 3.12 being the -primary version, you would execute ``make install`` in your 3.12 build directory +For example, if you want to install Python 2.7, 3.6, and 3.13 with 3.13 being the +primary version, you would execute ``make install`` in your 3.13 build directory and ``make altinstall`` in the others. @@ -238,7 +238,7 @@ All current PEPs, as well as guidelines for submitting a new PEP, are listed at Release Schedule ---------------- -See :pep:`693` for Python 3.12 release details. +See :pep:`XXX` for Python 3.13 release details. Copyright and License Information diff --git a/configure.ac b/configure.ac index 786d3414eb0e06..d5cb822ae8cb6b 100644 --- a/configure.ac +++ b/configure.ac @@ -6,7 +6,7 @@ dnl Python's configure.ac file requires autoconf 2.69 and autoconf-archive. dnl # Set VERSION so we only need to edit in one place (i.e., here) -m4_define(PYTHON_VERSION, 3.12) +m4_define(PYTHON_VERSION, 3.13) AC_PREREQ([2.69]) @@ -202,7 +202,7 @@ AC_SUBST([FREEZE_MODULE_DEPS]) AC_SUBST([PYTHON_FOR_BUILD_DEPS]) AC_CHECK_PROGS([PYTHON_FOR_REGEN], - [python$PACKAGE_VERSION python3.12 python3.11 python3.10 python3 python], + [python$PACKAGE_VERSION python3.13 python3.12 python3.11 python3.10 python3 python], [python3]) AC_SUBST(PYTHON_FOR_REGEN) From 357bed0bcd3c5d7c4a8caad451754a9a172aca3e Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Mon, 22 May 2023 12:34:34 -0700 Subject: [PATCH 25/47] GH-104668: Don't call PyOS_* hooks in subinterpreters (GH-104674) --- Doc/c-api/veryhigh.rst | 8 +++++ Doc/whatsnew/3.12.rst | 9 ++++++ ...-05-19-10-22-34.gh-issue-104668.MLX1g9.rst | 5 +++ Parser/myreadline.c | 32 +++++++++++++++---- 4 files changed, 47 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-05-19-10-22-34.gh-issue-104668.MLX1g9.rst diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 513856d8a48d70..000a2d3d8790bb 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -167,6 +167,10 @@ the same library that the Python runtime is using. event loops, as done in the :file:`Modules/_tkinter.c` in the Python source code. + .. versionchanged:: 3.12 + This function is only called from the + :ref:`main interpreter `. + .. c:var:: char* (*PyOS_ReadlineFunctionPointer)(FILE *, FILE *, const char *) @@ -187,6 +191,10 @@ the same library that the Python runtime is using. :c:func:`PyMem_RawRealloc`, instead of being allocated by :c:func:`PyMem_Malloc` or :c:func:`PyMem_Realloc`. + .. versionchanged:: 3.12 + This function is only called from the + :ref:`main interpreter `. + .. c:function:: PyObject* PyRun_String(const char *str, int start, PyObject *globals, PyObject *locals) This is a simplified interface to :c:func:`PyRun_StringFlags` below, leaving diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index efbd2ca3de122a..4ff90664bb790b 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1476,6 +1476,15 @@ Porting to Python 3.12 Note that :c:func:`PyType_FromMetaclass` (added in Python 3.12) already disallows creating classes whose metaclass overrides ``tp_new``. +* :c:var:`PyOS_InputHook` and :c:var:`PyOS_ReadlineFunctionPointer` are no + longer called in :ref:`subinterpreters `. This is + because clients generally rely on process-wide global state (since these + callbacks have no way of recovering extension module state). + + This also avoids situations where extensions may find themselves running in a + subinterpreter that they don't support (or haven't yet been loaded in). See + :gh:`104668` for more info. + Deprecated ---------- diff --git a/Misc/NEWS.d/next/C API/2023-05-19-10-22-34.gh-issue-104668.MLX1g9.rst b/Misc/NEWS.d/next/C API/2023-05-19-10-22-34.gh-issue-104668.MLX1g9.rst new file mode 100644 index 00000000000000..7b882afd7f81a0 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-05-19-10-22-34.gh-issue-104668.MLX1g9.rst @@ -0,0 +1,5 @@ +Don't call :c:var:`PyOS_InputHook` or :c:var:`PyOS_ReadlineFunctionPointer` +in subinterpreters, since it's generally difficult to avoid using global +state in their registered callbacks. This also avoids situations where +extensions may find themselves running in a subinterpreter they don't +support (or haven't yet been loaded in). diff --git a/Parser/myreadline.c b/Parser/myreadline.c index 3f0e29f051a438..7074aba74b728c 100644 --- a/Parser/myreadline.c +++ b/Parser/myreadline.c @@ -45,7 +45,10 @@ my_fgets(PyThreadState* tstate, char *buf, int len, FILE *fp) #endif while (1) { - if (PyOS_InputHook != NULL) { + if (PyOS_InputHook != NULL && + // GH-104668: See PyOS_ReadlineFunctionPointer's comment below... + _Py_IsMainInterpreter(tstate->interp)) + { (void)(PyOS_InputHook)(); } @@ -131,7 +134,10 @@ _PyOS_WindowsConsoleReadline(PyThreadState *tstate, HANDLE hStdIn) wbuf = wbuf_local; wbuflen = sizeof(wbuf_local) / sizeof(wbuf_local[0]) - 1; while (1) { - if (PyOS_InputHook != NULL) { + if (PyOS_InputHook != NULL && + // GH-104668: See PyOS_ReadlineFunctionPointer's comment below... + _Py_IsMainInterpreter(tstate->interp)) + { (void)(PyOS_InputHook)(); } if (!ReadConsoleW(hStdIn, &wbuf[total_read], wbuflen - total_read, &n_read, NULL)) { @@ -389,11 +395,23 @@ PyOS_Readline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt) * a tty. This can happen, for example if python is run like * this: python -i < test1.py */ - if (!isatty (fileno (sys_stdin)) || !isatty (fileno (sys_stdout))) - rv = PyOS_StdioReadline (sys_stdin, sys_stdout, prompt); - else - rv = (*PyOS_ReadlineFunctionPointer)(sys_stdin, sys_stdout, - prompt); + if (!isatty(fileno(sys_stdin)) || !isatty(fileno(sys_stdout)) || + // GH-104668: Don't call global callbacks like PyOS_InputHook or + // PyOS_ReadlineFunctionPointer from subinterpreters, since it seems + // like there's no good way for users (like readline and tkinter) to + // avoid using global state to manage them. Plus, we generally don't + // want to cause trouble for libraries that don't know/care about + // subinterpreter support. If libraries really need better APIs that + // work per-interpreter and have ways to access module state, we can + // certainly add them later (but for now we'll cross our fingers and + // hope that nobody actually cares): + !_Py_IsMainInterpreter(tstate->interp)) + { + rv = PyOS_StdioReadline(sys_stdin, sys_stdout, prompt); + } + else { + rv = (*PyOS_ReadlineFunctionPointer)(sys_stdin, sys_stdout, prompt); + } Py_END_ALLOW_THREADS PyThread_release_lock(_PyOS_ReadlineLock); From 42974c46bc9b1f282d35cf786b45f604b4df1bda Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Mon, 22 May 2023 16:05:27 -0400 Subject: [PATCH 26/47] Fix whatsnew for 3.13. (GH-104756) --- Doc/whatsnew/{3.13 => 3.13.rst} | 0 Doc/whatsnew/index.rst | 1 + 2 files changed, 1 insertion(+) rename Doc/whatsnew/{3.13 => 3.13.rst} (100%) diff --git a/Doc/whatsnew/3.13 b/Doc/whatsnew/3.13.rst similarity index 100% rename from Doc/whatsnew/3.13 rename to Doc/whatsnew/3.13.rst diff --git a/Doc/whatsnew/index.rst b/Doc/whatsnew/index.rst index bfee225791eee9..b9c19602653219 100644 --- a/Doc/whatsnew/index.rst +++ b/Doc/whatsnew/index.rst @@ -11,6 +11,7 @@ anyone wishing to stay up-to-date after a new release. .. toctree:: :maxdepth: 2 + 3.13.rst 3.12.rst 3.11.rst 3.10.rst From bc285c667c909b0690f659be3cbd96bb837d790a Mon Sep 17 00:00:00 2001 From: "T. Wouters" Date: Mon, 22 May 2023 23:50:26 +0200 Subject: [PATCH 27/47] Regenerate configure after the Python version bump (#104757) Regenerate configure after the Python version bump. --- configure | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/configure b/configure index 2b863be108be26..bcf2a5c7c7f73c 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for python 3.12. +# Generated by GNU Autoconf 2.69 for python 3.13. # # Report bugs to . # @@ -580,8 +580,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='python' PACKAGE_TARNAME='python' -PACKAGE_VERSION='3.12' -PACKAGE_STRING='python 3.12' +PACKAGE_VERSION='3.13' +PACKAGE_STRING='python 3.13' PACKAGE_BUGREPORT='https://github.com/python/cpython/issues/' PACKAGE_URL='' @@ -1690,7 +1690,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures python 3.12 to adapt to many kinds of systems. +\`configure' configures python 3.13 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1756,7 +1756,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of python 3.12:";; + short | recursive ) echo "Configuration of python 3.13:";; esac cat <<\_ACEOF @@ -1800,9 +1800,9 @@ Optional Features: Optional Packages: --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) - --with-build-python=python3.12 + --with-build-python=python3.13 path to build python binary for cross compiling - (default: _bootstrap_python or python3.12) + (default: _bootstrap_python or python3.13) --with-pkg-config=[yes|no|check] use pkg-config to detect build options (default is check) @@ -2041,7 +2041,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -python configure 3.12 +python configure 3.13 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2750,7 +2750,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by python $as_me 3.12, which was +It was created by python $as_me 3.13, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -3366,7 +3366,7 @@ fi -for ac_prog in python$PACKAGE_VERSION python3.12 python3.11 python3.10 python3 python +for ac_prog in python$PACKAGE_VERSION python3.13 python3.12 python3.11 python3.10 python3 python do # Extract the first word of "$ac_prog", so it can be a program name with args. set dummy $ac_prog; ac_word=$2 @@ -3436,7 +3436,7 @@ rm confdefs.h mv confdefs.h.new confdefs.h -VERSION=3.12 +VERSION=3.13 # Version number of Python's own shared library file. @@ -28971,7 +28971,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by python $as_me 3.12, which was +This file was extended by python $as_me 3.13, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -29033,7 +29033,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -python config.status 3.12 +python config.status 3.13 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" From 4ded2c5e9c92cecb759c306470b4d18c99b391bf Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Tue, 23 May 2023 01:20:06 +0300 Subject: [PATCH 28/47] Update Windows library names for the Python version bump (#104755) --- PC/pyconfig.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PC/pyconfig.h b/PC/pyconfig.h index 3415efe2dea117..f9250d6faccbea 100644 --- a/PC/pyconfig.h +++ b/PC/pyconfig.h @@ -308,11 +308,11 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ file in their Makefile (other compilers are generally taken care of by distutils.) */ # if defined(_DEBUG) -# pragma comment(lib,"python312_d.lib") +# pragma comment(lib,"python313_d.lib") # elif defined(Py_LIMITED_API) # pragma comment(lib,"python3.lib") # else -# pragma comment(lib,"python312.lib") +# pragma comment(lib,"python313.lib") # endif /* _DEBUG */ # endif /* _MSC_VER */ # endif /* Py_BUILD_CORE */ From 421cbf39fba94c4031c87d3c3e9bb8a8a4ca54a8 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 22 May 2023 17:03:17 -0600 Subject: [PATCH 29/47] gh-102304: Temporarily Bump Py_LIMITED_API for 2 New Functions (#104766) Quick and dirty. --- Include/object.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/object.h b/Include/object.h index 81aeb2d8bd5a69..c4fe2f83ef62bc 100644 --- a/Include/object.h +++ b/Include/object.h @@ -590,7 +590,7 @@ you can count such references to the type object.) extern Py_ssize_t _Py_RefTotal; # define _Py_INC_REFTOTAL() _Py_RefTotal++ # define _Py_DEC_REFTOTAL() _Py_RefTotal-- -# elif !defined(Py_LIMITED_API) || Py_LIMITED_API+0 > 0x030C0000 +# elif !defined(Py_LIMITED_API) || Py_LIMITED_API+0 > 0x030D0000 PyAPI_FUNC(void) _Py_IncRefTotal_DO_NOT_USE_THIS(void); PyAPI_FUNC(void) _Py_DecRefTotal_DO_NOT_USE_THIS(void); # define _Py_INC_REFTOTAL() _Py_IncRefTotal_DO_NOT_USE_THIS() From 4194d8f2c40f478eb0fc9b6fa9b913baaff229da Mon Sep 17 00:00:00 2001 From: "T. Wouters" Date: Tue, 23 May 2023 01:16:28 +0200 Subject: [PATCH 30/47] Add the 3.12 branch to the github workflows. (#104768) --- .github/workflows/build.yml | 2 ++ .github/workflows/doc.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 41abddffa5d648..b9797192dba054 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,6 +8,7 @@ on: push: branches: - 'main' + - '3.12' - '3.11' - '3.10' - '3.9' @@ -16,6 +17,7 @@ on: pull_request: branches: - 'main' + - '3.12' - '3.11' - '3.10' - '3.9' diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 3f7550cc72943b..ec900ce68a1dde 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -5,6 +5,7 @@ on: #push: # branches: # - 'main' + # - '3.12' # - '3.11' # - '3.10' # - '3.9' @@ -15,6 +16,7 @@ on: pull_request: branches: - 'main' + - '3.12' - '3.11' - '3.10' - '3.9' From 586aca3fc647c626c39e995d07d8bd7dd13e2d52 Mon Sep 17 00:00:00 2001 From: "T. Wouters" Date: Tue, 23 May 2023 01:17:06 +0200 Subject: [PATCH 31/47] gh-104764: Skip failing tests in test_enum that started failing. (#104765) Skip failing tests in test_enum that started failing when the version was bumped to 3.13. --- Lib/test/test_enum.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index fb7a016c9007f8..350554bfd2cc5b 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -1090,7 +1090,7 @@ class Inner(Enum): ) @unittest.skipIf( - python_version < (3, 13), + python_version < (3, 14), 'inner classes are still members', ) def test_nested_classes_in_enum_are_not_members(self): @@ -4261,8 +4261,8 @@ class Color(Enum): self.assertEqual(Color.green.value, 3) @unittest.skipIf( - python_version < (3, 13), - 'mixed types with auto() will raise in 3.13', + python_version < (3, 14), + 'mixed types with auto() will raise in the future', ) def test_auto_garbage_fail(self): with self.assertRaisesRegex(TypeError, 'will require all values to be sortable'): @@ -4271,8 +4271,8 @@ class Color(Enum): blue = auto() @unittest.skipIf( - python_version < (3, 13), - 'mixed types with auto() will raise in 3.13', + python_version < (3, 14), + 'mixed types with auto() will raise in the future', ) def test_auto_garbage_corrected_fail(self): with self.assertRaisesRegex(TypeError, 'will require all values to be sortable'): @@ -4303,8 +4303,8 @@ def _generate_next_value_(name, start, count, last): self.assertEqual(Color.blue.value, 'blue') @unittest.skipIf( - python_version < (3, 13), - 'auto() will return highest value + 1 in 3.13', + python_version < (3, 14), + 'auto() will return highest value + 1 in the future', ) def test_auto_with_aliases(self): class Color(Enum): From 988c1f68ce7f99f43322722c4a4f8f85b40bbcd8 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 23 May 2023 00:27:10 +0100 Subject: [PATCH 32/47] Add codeowners for the tokenizer module (#104753) --- .github/CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 32ba5355a5853f..d218f029a990bf 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -97,6 +97,8 @@ Include/pytime.h @pganssle @abalkin /Tools/peg_generator/ @pablogsal @lysnikolaou /Lib/test/test_peg_generator/ @pablogsal @lysnikolaou /Grammar/python.gram @pablogsal @lysnikolaou +/Lib/tokenize.py @pablogsal @lysnikolaou +/Lib/test/test_tokenize.py @pablogsal @lysnikolaou # AST Python/ast.c @isidentical From 2e5d8a90aa633ff0bebc9b2b8e21eea389937b19 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Mon, 22 May 2023 17:06:41 -0700 Subject: [PATCH 33/47] gh-99108: Release the GIL around hashlib built-in computation (#104675) This matches the GIL releasing behavior of our existing `_hashopenssl` module, extending it to the HACL* built-ins. Includes adding comments to better describe the ENTER/LEAVE macros purpose and explain the lock strategy in both existing and new code. --- ...3-05-19-19-46-22.gh-issue-99108.wqCg0t.rst | 3 + Modules/_hashopenssl.c | 6 ++ Modules/hashlib.h | 9 +- Modules/md5module.c | 37 +++++++- Modules/sha1module.c | 37 +++++++- Modules/sha2module.c | 94 +++++++++++++++++-- Modules/sha3module.c | 36 ++++++- 7 files changed, 207 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-05-19-19-46-22.gh-issue-99108.wqCg0t.rst diff --git a/Misc/NEWS.d/next/Library/2023-05-19-19-46-22.gh-issue-99108.wqCg0t.rst b/Misc/NEWS.d/next/Library/2023-05-19-19-46-22.gh-issue-99108.wqCg0t.rst new file mode 100644 index 00000000000000..b595f1893609cc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-19-19-46-22.gh-issue-99108.wqCg0t.rst @@ -0,0 +1,3 @@ +We now release the GIL around built-in :mod:`hashlib` computations of +reasonable size for the SHA families and MD5 hash functions, matching +what our OpenSSL backed hash computations already does. diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 99d0b72819137e..4b425f4147513e 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -227,12 +227,16 @@ get_hashlib_state(PyObject *module) typedef struct { PyObject_HEAD EVP_MD_CTX *ctx; /* OpenSSL message digest context */ + // Prevents undefined behavior via multiple threads entering the C API. + // The lock will be NULL before threaded access has been enabled. PyThread_type_lock lock; /* OpenSSL context lock */ } EVPobject; typedef struct { PyObject_HEAD HMAC_CTX *ctx; /* OpenSSL hmac context */ + // Prevents undefined behavior via multiple threads entering the C API. + // The lock will be NULL before threaded access has been enabled. PyThread_type_lock lock; /* HMAC context lock */ } HMACobject; @@ -896,6 +900,8 @@ py_evp_fromname(PyObject *module, const char *digestname, PyObject *data_obj, if (view.buf && view.len) { if (view.len >= HASHLIB_GIL_MINSIZE) { + /* We do not initialize self->lock here as this is the constructor + * where it is not yet possible to have concurrent access. */ Py_BEGIN_ALLOW_THREADS result = EVP_hash(self, view.buf, view.len); Py_END_ALLOW_THREADS diff --git a/Modules/hashlib.h b/Modules/hashlib.h index 56ae7a5e50bf58..a8bad9dd87a939 100644 --- a/Modules/hashlib.h +++ b/Modules/hashlib.h @@ -37,6 +37,13 @@ * LEAVE_HASHLIB block or explicitly acquire and release the lock inside * a PY_BEGIN / END_ALLOW_THREADS block if they wish to release the GIL for * an operation. + * + * These only drop the GIL if the lock acquisition itself is likely to + * block. Thus the non-blocking acquire gating the GIL release for a + * blocking lock acquisition. The intent of these macros is to surround + * the assumed always "fast" operations that you aren't releasing the + * GIL around. Otherwise use code similar to what you see in hash + * function update() methods. */ #include "pythread.h" @@ -53,7 +60,7 @@ PyThread_release_lock((obj)->lock); \ } -/* TODO(gps): We should probably make this a module or EVPobject attribute +/* TODO(gpshead): We should make this a module or class attribute * to allow the user to optimize based on the platform they're using. */ #define HASHLIB_GIL_MINSIZE 2048 diff --git a/Modules/md5module.c b/Modules/md5module.c index 86605771d9643f..2122f8b18baf6e 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -49,7 +49,9 @@ typedef long long MD5_INT64; /* 64-bit integer */ typedef struct { PyObject_HEAD - + // Prevents undefined behavior via multiple threads entering the C API. + // The lock will be NULL before threaded access has been enabled. + PyThread_type_lock lock; Hacl_Streaming_MD5_state *hash_state; } MD5object; @@ -72,6 +74,7 @@ static MD5object * newMD5object(MD5State * st) { MD5object *md5 = (MD5object *)PyObject_GC_New(MD5object, st->md5_type); + md5->lock = NULL; PyObject_GC_Track(md5); return md5; } @@ -88,6 +91,9 @@ static void MD5_dealloc(MD5object *ptr) { Hacl_Streaming_MD5_legacy_free(ptr->hash_state); + if (ptr->lock != NULL) { + PyThread_free_lock(ptr->lock); + } PyTypeObject *tp = Py_TYPE(ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); @@ -115,7 +121,9 @@ MD5Type_copy_impl(MD5object *self, PyTypeObject *cls) if ((newobj = newMD5object(st))==NULL) return NULL; + ENTER_HASHLIB(self); newobj->hash_state = Hacl_Streaming_MD5_legacy_copy(self->hash_state); + LEAVE_HASHLIB(self); return (PyObject *)newobj; } @@ -130,7 +138,9 @@ MD5Type_digest_impl(MD5object *self) /*[clinic end generated code: output=eb691dc4190a07ec input=bc0c4397c2994be6]*/ { unsigned char digest[MD5_DIGESTSIZE]; + ENTER_HASHLIB(self); Hacl_Streaming_MD5_legacy_finish(self->hash_state, digest); + LEAVE_HASHLIB(self); return PyBytes_FromStringAndSize((const char *)digest, MD5_DIGESTSIZE); } @@ -145,7 +155,9 @@ MD5Type_hexdigest_impl(MD5object *self) /*[clinic end generated code: output=17badced1f3ac932 input=b60b19de644798dd]*/ { unsigned char digest[MD5_DIGESTSIZE]; + ENTER_HASHLIB(self); Hacl_Streaming_MD5_legacy_finish(self->hash_state, digest); + LEAVE_HASHLIB(self); return _Py_strhex((const char*)digest, MD5_DIGESTSIZE); } @@ -177,7 +189,18 @@ MD5Type_update(MD5object *self, PyObject *obj) GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - update(self->hash_state, buf.buf, buf.len); + if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) { + self->lock = PyThread_allocate_lock(); + } + if (self->lock != NULL) { + Py_BEGIN_ALLOW_THREADS + PyThread_acquire_lock(self->lock, 1); + update(self->hash_state, buf.buf, buf.len); + PyThread_release_lock(self->lock); + Py_END_ALLOW_THREADS + } else { + update(self->hash_state, buf.buf, buf.len); + } PyBuffer_Release(&buf); Py_RETURN_NONE; @@ -279,7 +302,15 @@ _md5_md5_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } if (string) { - update(new->hash_state, buf.buf, buf.len); + if (buf.len >= HASHLIB_GIL_MINSIZE) { + /* We do not initialize self->lock here as this is the constructor + * where it is not yet possible to have concurrent access. */ + Py_BEGIN_ALLOW_THREADS + update(new->hash_state, buf.buf, buf.len); + Py_END_ALLOW_THREADS + } else { + update(new->hash_state, buf.buf, buf.len); + } PyBuffer_Release(&buf); } diff --git a/Modules/sha1module.c b/Modules/sha1module.c index bdb76c56f1a6e8..c66269b5f5cdf3 100644 --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -48,7 +48,9 @@ typedef long long SHA1_INT64; /* 64-bit integer */ typedef struct { PyObject_HEAD - + // Prevents undefined behavior via multiple threads entering the C API. + // The lock will be NULL before threaded access has been enabled. + PyThread_type_lock lock; Hacl_Streaming_SHA1_state *hash_state; } SHA1object; @@ -71,6 +73,7 @@ static SHA1object * newSHA1object(SHA1State *st) { SHA1object *sha = (SHA1object *)PyObject_GC_New(SHA1object, st->sha1_type); + sha->lock = NULL; PyObject_GC_Track(sha); return sha; } @@ -88,6 +91,9 @@ static void SHA1_dealloc(SHA1object *ptr) { Hacl_Streaming_SHA1_legacy_free(ptr->hash_state); + if (ptr->lock != NULL) { + PyThread_free_lock(ptr->lock); + } PyTypeObject *tp = Py_TYPE(ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); @@ -115,7 +121,9 @@ SHA1Type_copy_impl(SHA1object *self, PyTypeObject *cls) if ((newobj = newSHA1object(st)) == NULL) return NULL; + ENTER_HASHLIB(self); newobj->hash_state = Hacl_Streaming_SHA1_legacy_copy(self->hash_state); + LEAVE_HASHLIB(self); return (PyObject *)newobj; } @@ -130,7 +138,9 @@ SHA1Type_digest_impl(SHA1object *self) /*[clinic end generated code: output=2f05302a7aa2b5cb input=13824b35407444bd]*/ { unsigned char digest[SHA1_DIGESTSIZE]; + ENTER_HASHLIB(self); Hacl_Streaming_SHA1_legacy_finish(self->hash_state, digest); + LEAVE_HASHLIB(self); return PyBytes_FromStringAndSize((const char *)digest, SHA1_DIGESTSIZE); } @@ -145,7 +155,9 @@ SHA1Type_hexdigest_impl(SHA1object *self) /*[clinic end generated code: output=4161fd71e68c6659 input=97691055c0c74ab0]*/ { unsigned char digest[SHA1_DIGESTSIZE]; + ENTER_HASHLIB(self); Hacl_Streaming_SHA1_legacy_finish(self->hash_state, digest); + LEAVE_HASHLIB(self); return _Py_strhex((const char *)digest, SHA1_DIGESTSIZE); } @@ -177,7 +189,18 @@ SHA1Type_update(SHA1object *self, PyObject *obj) GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - update(self->hash_state, buf.buf, buf.len); + if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) { + self->lock = PyThread_allocate_lock(); + } + if (self->lock != NULL) { + Py_BEGIN_ALLOW_THREADS + PyThread_acquire_lock(self->lock, 1); + update(self->hash_state, buf.buf, buf.len); + PyThread_release_lock(self->lock); + Py_END_ALLOW_THREADS + } else { + update(self->hash_state, buf.buf, buf.len); + } PyBuffer_Release(&buf); Py_RETURN_NONE; @@ -279,7 +302,15 @@ _sha1_sha1_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } if (string) { - update(new->hash_state, buf.buf, buf.len); + if (buf.len >= HASHLIB_GIL_MINSIZE) { + /* We do not initialize self->lock here as this is the constructor + * where it is not yet possible to have concurrent access. */ + Py_BEGIN_ALLOW_THREADS + update(new->hash_state, buf.buf, buf.len); + Py_END_ALLOW_THREADS + } else { + update(new->hash_state, buf.buf, buf.len); + } PyBuffer_Release(&buf); } diff --git a/Modules/sha2module.c b/Modules/sha2module.c index 37d9b5c538fd0b..6c7c3917198d18 100644 --- a/Modules/sha2module.c +++ b/Modules/sha2module.c @@ -52,12 +52,18 @@ class SHA512Type "SHA512object *" "&PyType_Type" typedef struct { PyObject_HEAD int digestsize; + // Prevents undefined behavior via multiple threads entering the C API. + // The lock will be NULL before threaded access has been enabled. + PyThread_type_lock lock; Hacl_Streaming_SHA2_state_sha2_256 *state; } SHA256object; typedef struct { PyObject_HEAD int digestsize; + // Prevents undefined behavior via multiple threads entering the C API. + // The lock will be NULL before threaded access has been enabled. + PyThread_type_lock lock; Hacl_Streaming_SHA2_state_sha2_512 *state; } SHA512object; @@ -100,6 +106,7 @@ newSHA224object(sha2_state *state) if (!sha) { return NULL; } + sha->lock = NULL; PyObject_GC_Track(sha); return sha; } @@ -112,6 +119,7 @@ newSHA256object(sha2_state *state) if (!sha) { return NULL; } + sha->lock = NULL; PyObject_GC_Track(sha); return sha; } @@ -124,6 +132,7 @@ newSHA384object(sha2_state *state) if (!sha) { return NULL; } + sha->lock = NULL; PyObject_GC_Track(sha); return sha; } @@ -136,6 +145,7 @@ newSHA512object(sha2_state *state) if (!sha) { return NULL; } + sha->lock = NULL; PyObject_GC_Track(sha); return sha; } @@ -153,6 +163,9 @@ static void SHA256_dealloc(SHA256object *ptr) { Hacl_Streaming_SHA2_free_256(ptr->state); + if (ptr->lock != NULL) { + PyThread_free_lock(ptr->lock); + } PyTypeObject *tp = Py_TYPE(ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); @@ -163,6 +176,9 @@ static void SHA512_dealloc(SHA512object *ptr) { Hacl_Streaming_SHA2_free_512(ptr->state); + if (ptr->lock != NULL) { + PyThread_free_lock(ptr->lock); + } PyTypeObject *tp = Py_TYPE(ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); @@ -229,7 +245,9 @@ SHA256Type_copy_impl(SHA256object *self, PyTypeObject *cls) } } + ENTER_HASHLIB(self); SHA256copy(self, newobj); + LEAVE_HASHLIB(self); return (PyObject *)newobj; } @@ -259,7 +277,9 @@ SHA512Type_copy_impl(SHA512object *self, PyTypeObject *cls) } } + ENTER_HASHLIB(self); SHA512copy(self, newobj); + LEAVE_HASHLIB(self); return (PyObject *)newobj; } @@ -275,9 +295,11 @@ SHA256Type_digest_impl(SHA256object *self) { uint8_t digest[SHA256_DIGESTSIZE]; assert(self->digestsize <= SHA256_DIGESTSIZE); + ENTER_HASHLIB(self); // HACL* performs copies under the hood so that self->state remains valid // after this call. Hacl_Streaming_SHA2_finish_256(self->state, digest); + LEAVE_HASHLIB(self); return PyBytes_FromStringAndSize((const char *)digest, self->digestsize); } @@ -293,9 +315,11 @@ SHA512Type_digest_impl(SHA512object *self) { uint8_t digest[SHA512_DIGESTSIZE]; assert(self->digestsize <= SHA512_DIGESTSIZE); + ENTER_HASHLIB(self); // HACL* performs copies under the hood so that self->state remains valid // after this call. Hacl_Streaming_SHA2_finish_512(self->state, digest); + LEAVE_HASHLIB(self); return PyBytes_FromStringAndSize((const char *)digest, self->digestsize); } @@ -311,7 +335,9 @@ SHA256Type_hexdigest_impl(SHA256object *self) { uint8_t digest[SHA256_DIGESTSIZE]; assert(self->digestsize <= SHA256_DIGESTSIZE); + ENTER_HASHLIB(self); Hacl_Streaming_SHA2_finish_256(self->state, digest); + LEAVE_HASHLIB(self); return _Py_strhex((const char *)digest, self->digestsize); } @@ -327,7 +353,9 @@ SHA512Type_hexdigest_impl(SHA512object *self) { uint8_t digest[SHA512_DIGESTSIZE]; assert(self->digestsize <= SHA512_DIGESTSIZE); + ENTER_HASHLIB(self); Hacl_Streaming_SHA2_finish_512(self->state, digest); + LEAVE_HASHLIB(self); return _Py_strhex((const char *)digest, self->digestsize); } @@ -348,7 +376,18 @@ SHA256Type_update(SHA256object *self, PyObject *obj) GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - update_256(self->state, buf.buf, buf.len); + if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) { + self->lock = PyThread_allocate_lock(); + } + if (self->lock != NULL) { + Py_BEGIN_ALLOW_THREADS + PyThread_acquire_lock(self->lock, 1); + update_256(self->state, buf.buf, buf.len); + PyThread_release_lock(self->lock); + Py_END_ALLOW_THREADS + } else { + update_256(self->state, buf.buf, buf.len); + } PyBuffer_Release(&buf); Py_RETURN_NONE; @@ -371,7 +410,18 @@ SHA512Type_update(SHA512object *self, PyObject *obj) GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - update_512(self->state, buf.buf, buf.len); + if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) { + self->lock = PyThread_allocate_lock(); + } + if (self->lock != NULL) { + Py_BEGIN_ALLOW_THREADS + PyThread_acquire_lock(self->lock, 1); + update_512(self->state, buf.buf, buf.len); + PyThread_release_lock(self->lock); + Py_END_ALLOW_THREADS + } else { + update_512(self->state, buf.buf, buf.len); + } PyBuffer_Release(&buf); Py_RETURN_NONE; @@ -560,7 +610,15 @@ _sha2_sha256_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } if (string) { - update_256(new->state, buf.buf, buf.len); + if (buf.len >= HASHLIB_GIL_MINSIZE) { + /* We do not initialize self->lock here as this is the constructor + * where it is not yet possible to have concurrent access. */ + Py_BEGIN_ALLOW_THREADS + update_256(new->state, buf.buf, buf.len); + Py_END_ALLOW_THREADS + } else { + update_256(new->state, buf.buf, buf.len); + } PyBuffer_Release(&buf); } @@ -606,7 +664,15 @@ _sha2_sha224_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } if (string) { - update_256(new->state, buf.buf, buf.len); + if (buf.len >= HASHLIB_GIL_MINSIZE) { + /* We do not initialize self->lock here as this is the constructor + * where it is not yet possible to have concurrent access. */ + Py_BEGIN_ALLOW_THREADS + update_256(new->state, buf.buf, buf.len); + Py_END_ALLOW_THREADS + } else { + update_256(new->state, buf.buf, buf.len); + } PyBuffer_Release(&buf); } @@ -651,7 +717,15 @@ _sha2_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } if (string) { - update_512(new->state, buf.buf, buf.len); + if (buf.len >= HASHLIB_GIL_MINSIZE) { + /* We do not initialize self->lock here as this is the constructor + * where it is not yet possible to have concurrent access. */ + Py_BEGIN_ALLOW_THREADS + update_512(new->state, buf.buf, buf.len); + Py_END_ALLOW_THREADS + } else { + update_512(new->state, buf.buf, buf.len); + } PyBuffer_Release(&buf); } @@ -696,7 +770,15 @@ _sha2_sha384_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } if (string) { - update_512(new->state, buf.buf, buf.len); + if (buf.len >= HASHLIB_GIL_MINSIZE) { + /* We do not initialize self->lock here as this is the constructor + * where it is not yet possible to have concurrent access. */ + Py_BEGIN_ALLOW_THREADS + update_512(new->state, buf.buf, buf.len); + Py_END_ALLOW_THREADS + } else { + update_512(new->state, buf.buf, buf.len); + } PyBuffer_Release(&buf); } diff --git a/Modules/sha3module.c b/Modules/sha3module.c index f05187498a19b3..558d2005cff617 100644 --- a/Modules/sha3module.c +++ b/Modules/sha3module.c @@ -60,6 +60,9 @@ class _sha3.shake_256 "SHA3object *" "&SHAKE256type" typedef struct { PyObject_HEAD + // Prevents undefined behavior via multiple threads entering the C API. + // The lock will be NULL before threaded access has been enabled. + PyThread_type_lock lock; Hacl_Streaming_Keccak_state *hash_state; } SHA3object; @@ -73,6 +76,7 @@ newSHA3object(PyTypeObject *type) if (newobj == NULL) { return NULL; } + newobj->lock = NULL; return newobj; } @@ -133,7 +137,15 @@ py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity) if (data) { GET_BUFFER_VIEW_OR_ERROR(data, &buf, goto error); - sha3_update(self->hash_state, buf.buf, buf.len); + if (buf.len >= HASHLIB_GIL_MINSIZE) { + /* We do not initialize self->lock here as this is the constructor + * where it is not yet possible to have concurrent access. */ + Py_BEGIN_ALLOW_THREADS + sha3_update(self->hash_state, buf.buf, buf.len); + Py_END_ALLOW_THREADS + } else { + sha3_update(self->hash_state, buf.buf, buf.len); + } } PyBuffer_Release(&buf); @@ -157,6 +169,9 @@ static void SHA3_dealloc(SHA3object *self) { Hacl_Streaming_Keccak_free(self->hash_state); + if (self->lock != NULL) { + PyThread_free_lock(self->lock); + } PyTypeObject *tp = Py_TYPE(self); PyObject_Free(self); Py_DECREF(tp); @@ -181,7 +196,9 @@ _sha3_sha3_224_copy_impl(SHA3object *self) if ((newobj = newSHA3object(Py_TYPE(self))) == NULL) { return NULL; } + ENTER_HASHLIB(self); newobj->hash_state = Hacl_Streaming_Keccak_copy(self->hash_state); + LEAVE_HASHLIB(self); return (PyObject *)newobj; } @@ -199,7 +216,9 @@ _sha3_sha3_224_digest_impl(SHA3object *self) unsigned char digest[SHA3_MAX_DIGESTSIZE]; // This function errors out if the algorithm is Shake. Here, we know this // not to be the case, and therefore do not perform error checking. + ENTER_HASHLIB(self); Hacl_Streaming_Keccak_finish(self->hash_state, digest); + LEAVE_HASHLIB(self); return PyBytes_FromStringAndSize((const char *)digest, Hacl_Streaming_Keccak_hash_len(self->hash_state)); } @@ -216,7 +235,9 @@ _sha3_sha3_224_hexdigest_impl(SHA3object *self) /*[clinic end generated code: output=75ad03257906918d input=2d91bb6e0d114ee3]*/ { unsigned char digest[SHA3_MAX_DIGESTSIZE]; + ENTER_HASHLIB(self); Hacl_Streaming_Keccak_finish(self->hash_state, digest); + LEAVE_HASHLIB(self); return _Py_strhex((const char *)digest, Hacl_Streaming_Keccak_hash_len(self->hash_state)); } @@ -237,7 +258,18 @@ _sha3_sha3_224_update(SHA3object *self, PyObject *data) { Py_buffer buf; GET_BUFFER_VIEW_OR_ERROUT(data, &buf); - sha3_update(self->hash_state, buf.buf, buf.len); + if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) { + self->lock = PyThread_allocate_lock(); + } + if (self->lock != NULL) { + Py_BEGIN_ALLOW_THREADS + PyThread_acquire_lock(self->lock, 1); + sha3_update(self->hash_state, buf.buf, buf.len); + PyThread_release_lock(self->lock); + Py_END_ALLOW_THREADS + } else { + sha3_update(self->hash_state, buf.buf, buf.len); + } PyBuffer_Release(&buf); Py_RETURN_NONE; } From 5ecd8c85f934e13a5ff98db6539d89e0c7c03f2d Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Mon, 22 May 2023 18:59:40 -0700 Subject: [PATCH 34/47] gh-104764: [Enum] fix 3.13-specific tests (GH-104779) --- Lib/enum.py | 44 ++++++++++--------------------------------- Lib/test/test_enum.py | 18 +++++++++--------- 2 files changed, 19 insertions(+), 43 deletions(-) diff --git a/Lib/enum.py b/Lib/enum.py index 6e497f7ef6a7de..b50fe50d8258d3 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -388,16 +388,8 @@ def __setitem__(self, key, value): Single underscore (sunder) names are reserved. """ - if _is_internal_class(self._cls_name, value): - import warnings - warnings.warn( - "In 3.13 classes created inside an enum will not become a member. " - "Use the `member` decorator to keep the current behavior.", - DeprecationWarning, - stacklevel=2, - ) if _is_private(self._cls_name, key): - # also do nothing, name will be a normal attribute + # do nothing, name will be a normal attribute pass elif _is_sunder(key): if key not in ( @@ -440,10 +432,9 @@ def __setitem__(self, key, value): value = value.value elif _is_descriptor(value): pass - # TODO: uncomment next three lines in 3.13 - # elif _is_internal_class(self._cls_name, value): - # # do nothing, name will be a normal attribute - # pass + elif _is_internal_class(self._cls_name, value): + # do nothing, name will be a normal attribute + pass else: if key in self: # enum overwriting a descriptor? @@ -1169,28 +1160,13 @@ def _generate_next_value_(name, start, count, last_values): if not last_values: return start try: - last = last_values[-1] - last_values.sort() - if last == last_values[-1]: - # no difference between old and new methods - return last + 1 - else: - # trigger old method (with warning) - raise TypeError + last_value = sorted(last_values).pop() except TypeError: - import warnings - warnings.warn( - "In 3.13 the default `auto()`/`_generate_next_value_` will require all values to be sortable and support adding +1\n" - "and the value returned will be the largest value in the enum incremented by 1", - DeprecationWarning, - stacklevel=3, - ) - for v in last_values: - try: - return v + 1 - except TypeError: - pass - return start + raise TypeError('unable to sort non-numeric values') from None + try: + return last_value + 1 + except TypeError: + raise TypeError('unable to increment %r' % (last_value, )) from None @classmethod def _missing_(cls, value): diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 350554bfd2cc5b..fe701025b70f48 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -1090,7 +1090,7 @@ class Inner(Enum): ) @unittest.skipIf( - python_version < (3, 14), + python_version < (3, 13), 'inner classes are still members', ) def test_nested_classes_in_enum_are_not_members(self): @@ -4261,21 +4261,21 @@ class Color(Enum): self.assertEqual(Color.green.value, 3) @unittest.skipIf( - python_version < (3, 14), - 'mixed types with auto() will raise in the future', + python_version < (3, 13), + 'mixed types with auto() will raise in 3.13', ) def test_auto_garbage_fail(self): - with self.assertRaisesRegex(TypeError, 'will require all values to be sortable'): + with self.assertRaisesRegex(TypeError, "unable to increment 'red'"): class Color(Enum): red = 'red' blue = auto() @unittest.skipIf( - python_version < (3, 14), - 'mixed types with auto() will raise in the future', + python_version < (3, 13), + 'mixed types with auto() will raise in 3.13', ) def test_auto_garbage_corrected_fail(self): - with self.assertRaisesRegex(TypeError, 'will require all values to be sortable'): + with self.assertRaisesRegex(TypeError, 'unable to sort non-numeric values'): class Color(Enum): red = 'red' blue = 2 @@ -4303,8 +4303,8 @@ def _generate_next_value_(name, start, count, last): self.assertEqual(Color.blue.value, 'blue') @unittest.skipIf( - python_version < (3, 14), - 'auto() will return highest value + 1 in the future', + python_version < (3, 13), + 'auto() will return highest value + 1 in 3.13', ) def test_auto_with_aliases(self): class Color(Enum): From 9dc476be2dcfc9bcf53bcb83f4b8d555682d0600 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 23 May 2023 09:09:02 +0200 Subject: [PATCH 35/47] gh-104773: PEP 594: Remove the telnetlib module (#104778) --- Doc/library/superseded.rst | 1 - Doc/library/telnetlib.rst | 262 ------- Doc/tools/.nitignore | 1 - Doc/whatsnew/2.6.rst | 2 +- Doc/whatsnew/3.11.rst | 2 +- Doc/whatsnew/3.12.rst | 2 +- Doc/whatsnew/3.13.rst | 4 + Doc/whatsnew/3.6.rst | 2 +- Lib/telnetlib.py | 679 ------------------ Lib/test/test_telnetlib.py | 405 ----------- Misc/NEWS.d/3.9.0a1.rst | 2 +- ...-05-23-02-20-13.gh-issue-104773.7K59zr.rst | 2 + Python/stdlib_module_names.h | 1 - Tools/wasm/wasm_assets.py | 1 - 14 files changed, 11 insertions(+), 1355 deletions(-) delete mode 100644 Doc/library/telnetlib.rst delete mode 100644 Lib/telnetlib.py delete mode 100644 Lib/test/test_telnetlib.py create mode 100644 Misc/NEWS.d/next/Library/2023-05-23-02-20-13.gh-issue-104773.7K59zr.rst diff --git a/Doc/library/superseded.rst b/Doc/library/superseded.rst index aaf66ea121d39c..e4a473f712846d 100644 --- a/Doc/library/superseded.rst +++ b/Doc/library/superseded.rst @@ -27,6 +27,5 @@ backwards compatibility. They have been superseded by other modules. sndhdr.rst spwd.rst sunau.rst - telnetlib.rst uu.rst xdrlib.rst diff --git a/Doc/library/telnetlib.rst b/Doc/library/telnetlib.rst deleted file mode 100644 index 5a993dc42a5ab2..00000000000000 --- a/Doc/library/telnetlib.rst +++ /dev/null @@ -1,262 +0,0 @@ -:mod:`telnetlib` --- Telnet client -================================== - -.. module:: telnetlib - :synopsis: Telnet client class. - :deprecated: - -.. sectionauthor:: Skip Montanaro - -**Source code:** :source:`Lib/telnetlib.py` - -.. index:: single: protocol; Telnet - -.. deprecated-removed:: 3.11 3.13 - The :mod:`telnetlib` module is deprecated - (see :pep:`PEP 594 <594#telnetlib>` for details and alternatives). - --------------- - -The :mod:`telnetlib` module provides a :class:`Telnet` class that implements the -Telnet protocol. See :rfc:`854` for details about the protocol. In addition, it -provides symbolic constants for the protocol characters (see below), and for the -telnet options. The symbolic names of the telnet options follow the definitions -in ``arpa/telnet.h``, with the leading ``TELOPT_`` removed. For symbolic names -of options which are traditionally not included in ``arpa/telnet.h``, see the -module source itself. - -The symbolic constants for the telnet commands are: IAC, DONT, DO, WONT, WILL, -SE (Subnegotiation End), NOP (No Operation), DM (Data Mark), BRK (Break), IP -(Interrupt process), AO (Abort output), AYT (Are You There), EC (Erase -Character), EL (Erase Line), GA (Go Ahead), SB (Subnegotiation Begin). - -.. include:: ../includes/wasm-notavail.rst - -.. class:: Telnet(host=None, port=0[, timeout]) - - :class:`Telnet` represents a connection to a Telnet server. The instance is - initially not connected by default; the :meth:`~Telnet.open` method must be used to - establish a connection. Alternatively, the host name and optional port - number can be passed to the constructor too, in which case the connection to - the server will be established before the constructor returns. The optional - *timeout* parameter specifies a timeout in seconds for blocking operations - like the connection attempt (if not specified, the global default timeout - setting will be used). - - Do not reopen an already connected instance. - - This class has many :meth:`read_\*` methods. Note that some of them raise - :exc:`EOFError` when the end of the connection is read, because they can return - an empty string for other reasons. See the individual descriptions below. - - A :class:`Telnet` object is a context manager and can be used in a - :keyword:`with` statement. When the :keyword:`!with` block ends, the - :meth:`close` method is called:: - - >>> from telnetlib import Telnet - >>> with Telnet('localhost', 23) as tn: - ... tn.interact() - ... - - .. versionchanged:: 3.6 Context manager support added - - -.. seealso:: - - :rfc:`854` - Telnet Protocol Specification - Definition of the Telnet protocol. - - -.. _telnet-objects: - -Telnet Objects --------------- - -:class:`Telnet` instances have the following methods: - - -.. method:: Telnet.read_until(expected, timeout=None) - - Read until a given byte string, *expected*, is encountered or until *timeout* - seconds have passed. - - When no match is found, return whatever is available instead, possibly empty - bytes. Raise :exc:`EOFError` if the connection is closed and no cooked data - is available. - - -.. method:: Telnet.read_all() - - Read all data until EOF as bytes; block until connection closed. - - -.. method:: Telnet.read_some() - - Read at least one byte of cooked data unless EOF is hit. Return ``b''`` if - EOF is hit. Block if no data is immediately available. - - -.. method:: Telnet.read_very_eager() - - Read everything that can be without blocking in I/O (eager). - - Raise :exc:`EOFError` if connection closed and no cooked data available. - Return ``b''`` if no cooked data available otherwise. Do not block unless in - the midst of an IAC sequence. - - -.. method:: Telnet.read_eager() - - Read readily available data. - - Raise :exc:`EOFError` if connection closed and no cooked data available. - Return ``b''`` if no cooked data available otherwise. Do not block unless in - the midst of an IAC sequence. - - -.. method:: Telnet.read_lazy() - - Process and return data already in the queues (lazy). - - Raise :exc:`EOFError` if connection closed and no data available. Return - ``b''`` if no cooked data available otherwise. Do not block unless in the - midst of an IAC sequence. - - -.. method:: Telnet.read_very_lazy() - - Return any data available in the cooked queue (very lazy). - - Raise :exc:`EOFError` if connection closed and no data available. Return - ``b''`` if no cooked data available otherwise. This method never blocks. - - -.. method:: Telnet.read_sb_data() - - Return the data collected between a SB/SE pair (suboption begin/end). The - callback should access these data when it was invoked with a ``SE`` command. - This method never blocks. - - -.. method:: Telnet.open(host, port=0[, timeout]) - - Connect to a host. The optional second argument is the port number, which - defaults to the standard Telnet port (23). The optional *timeout* parameter - specifies a timeout in seconds for blocking operations like the connection - attempt (if not specified, the global default timeout setting will be used). - - Do not try to reopen an already connected instance. - - .. audit-event:: telnetlib.Telnet.open self,host,port telnetlib.Telnet.open - - -.. method:: Telnet.msg(msg, *args) - - Print a debug message when the debug level is ``>`` 0. If extra arguments are - present, they are substituted in the message using the standard string - formatting operator. - - -.. method:: Telnet.set_debuglevel(debuglevel) - - Set the debug level. The higher the value of *debuglevel*, the more debug - output you get (on ``sys.stdout``). - - -.. method:: Telnet.close() - - Close the connection. - - -.. method:: Telnet.get_socket() - - Return the socket object used internally. - - -.. method:: Telnet.fileno() - - Return the file descriptor of the socket object used internally. - - -.. method:: Telnet.write(buffer) - - Write a byte string to the socket, doubling any IAC characters. This can - block if the connection is blocked. May raise :exc:`OSError` if the - connection is closed. - - .. audit-event:: telnetlib.Telnet.write self,buffer telnetlib.Telnet.write - - .. versionchanged:: 3.3 - This method used to raise :exc:`socket.error`, which is now an alias - of :exc:`OSError`. - - -.. method:: Telnet.interact() - - Interaction function, emulates a very dumb Telnet client. - - -.. method:: Telnet.mt_interact() - - Multithreaded version of :meth:`interact`. - - -.. method:: Telnet.expect(list, timeout=None) - - Read until one from a list of a regular expressions matches. - - The first argument is a list of regular expressions, either compiled - (:ref:`regex objects `) or uncompiled (byte strings). The - optional second argument is a timeout, in seconds; the default is to block - indefinitely. - - Return a tuple of three items: the index in the list of the first regular - expression that matches; the match object returned; and the bytes read up - till and including the match. - - If end of file is found and no bytes were read, raise :exc:`EOFError`. - Otherwise, when nothing matches, return ``(-1, None, data)`` where *data* is - the bytes received so far (may be empty bytes if a timeout happened). - - If a regular expression ends with a greedy match (such as ``.*``) or if more - than one expression can match the same input, the results are - non-deterministic, and may depend on the I/O timing. - - -.. method:: Telnet.set_option_negotiation_callback(callback) - - Each time a telnet option is read on the input flow, this *callback* (if set) is - called with the following parameters: callback(telnet socket, command - (DO/DONT/WILL/WONT), option). No other action is done afterwards by telnetlib. - - -.. _telnet-example: - -Telnet Example --------------- - -.. sectionauthor:: Peter Funk - - -A simple example illustrating typical use:: - - import getpass - import telnetlib - - HOST = "localhost" - user = input("Enter your remote account: ") - password = getpass.getpass() - - tn = telnetlib.Telnet(HOST) - - tn.read_until(b"login: ") - tn.write(user.encode('ascii') + b"\n") - if password: - tn.read_until(b"Password: ") - tn.write(password.encode('ascii') + b"\n") - - tn.write(b"ls\n") - tn.write(b"exit\n") - - print(tn.read_all().decode('ascii')) - diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 1d3503bf06f085..554e31ff51dd3e 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -215,7 +215,6 @@ Doc/library/sys_path_init.rst Doc/library/sysconfig.rst Doc/library/syslog.rst Doc/library/tarfile.rst -Doc/library/telnetlib.rst Doc/library/tempfile.rst Doc/library/termios.rst Doc/library/test.rst diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 84bb651e68eed5..7236e1cad7c8c6 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -2465,7 +2465,7 @@ changes, or look through the Subversion logs for all the details. (All changes contributed by Lars Gustäbel). * An optional ``timeout`` parameter was added to the - :class:`telnetlib.Telnet` class constructor, specifying a timeout + :class:`!telnetlib.Telnet` class constructor, specifying a timeout measured in seconds. (Added by Facundo Batista.) * The :class:`tempfile.NamedTemporaryFile` class usually deletes diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 7a479c6e56a9f0..1a4a9936aca685 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -1731,7 +1731,7 @@ Modules slated for removal in Python 3.13: +---------------------+---------------------+---------------------+---------------------+---------------------+ - | :mod:`aifc` | :mod:`chunk` | :mod:`msilib` | :mod:`pipes` | :mod:`telnetlib` | + | :mod:`aifc` | :mod:`chunk` | :mod:`msilib` | :mod:`pipes` | :mod:`!telnetlib` | +---------------------+---------------------+---------------------+---------------------+---------------------+ | :mod:`audioop` | :mod:`crypt` | :mod:`nis` | :mod:`sndhdr` | :mod:`uu` | +---------------------+---------------------+---------------------+---------------------+---------------------+ diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 4ff90664bb790b..5e07a4caeb9ebe 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -819,7 +819,7 @@ Modules (see :pep:`594`): * :mod:`sndhdr` * :mod:`spwd` * :mod:`sunau` -* :mod:`telnetlib` +* :mod:`!telnetlib` * :mod:`uu` * :mod:`xdrlib` diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index af331c46f1c0e4..8256f429183a3a 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -102,6 +102,10 @@ Deprecated Removed ======= +* :pep:`594`: Remove the :mod:`!telnetlib` module, deprecated in Python 3.11: + use the projects `telnetlib3 `_ or + `Exscript `_ instead. + (Contributed by Victor Stinner in :gh:`104773`.) Porting to Python 3.13 diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 3a681754e25dd7..1cd33dfb60421a 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -1520,7 +1520,7 @@ rather than the version that is being emulated for the process telnetlib --------- -:class:`~telnetlib.Telnet` is now a context manager (contributed by +:class:`!telnetlib.Telnet` is now a context manager (contributed by Stéphane Wirtel in :issue:`25485`). diff --git a/Lib/telnetlib.py b/Lib/telnetlib.py deleted file mode 100644 index 62d636129853ad..00000000000000 --- a/Lib/telnetlib.py +++ /dev/null @@ -1,679 +0,0 @@ -r"""TELNET client class. - -Based on RFC 854: TELNET Protocol Specification, by J. Postel and -J. Reynolds - -Example: - ->>> from telnetlib import Telnet ->>> tn = Telnet('www.python.org', 79) # connect to finger port ->>> tn.write(b'guido\r\n') ->>> print(tn.read_all()) -Login Name TTY Idle When Where -guido Guido van Rossum pts/2 snag.cnri.reston.. - ->>> - -Note that read_all() won't read until eof -- it just reads some data --- but it guarantees to read at least one byte unless EOF is hit. - -It is possible to pass a Telnet object to a selector in order to wait until -more data is available. Note that in this case, read_eager() may return b'' -even if there was data on the socket, because the protocol negotiation may have -eaten the data. This is why EOFError is needed in some cases to distinguish -between "no data" and "connection closed" (since the socket also appears ready -for reading when it is closed). - -To do: -- option negotiation -- timeout should be intrinsic to the connection object instead of an - option on one of the read calls only - -""" - - -# Imported modules -import sys -import socket -import selectors -from time import monotonic as _time -import warnings - -warnings._deprecated(__name__, remove=(3, 13)) - -__all__ = ["Telnet"] - -# Tunable parameters -DEBUGLEVEL = 0 - -# Telnet protocol defaults -TELNET_PORT = 23 - -# Telnet protocol characters (don't change) -IAC = bytes([255]) # "Interpret As Command" -DONT = bytes([254]) -DO = bytes([253]) -WONT = bytes([252]) -WILL = bytes([251]) -theNULL = bytes([0]) - -SE = bytes([240]) # Subnegotiation End -NOP = bytes([241]) # No Operation -DM = bytes([242]) # Data Mark -BRK = bytes([243]) # Break -IP = bytes([244]) # Interrupt process -AO = bytes([245]) # Abort output -AYT = bytes([246]) # Are You There -EC = bytes([247]) # Erase Character -EL = bytes([248]) # Erase Line -GA = bytes([249]) # Go Ahead -SB = bytes([250]) # Subnegotiation Begin - - -# Telnet protocol options code (don't change) -# These ones all come from arpa/telnet.h -BINARY = bytes([0]) # 8-bit data path -ECHO = bytes([1]) # echo -RCP = bytes([2]) # prepare to reconnect -SGA = bytes([3]) # suppress go ahead -NAMS = bytes([4]) # approximate message size -STATUS = bytes([5]) # give status -TM = bytes([6]) # timing mark -RCTE = bytes([7]) # remote controlled transmission and echo -NAOL = bytes([8]) # negotiate about output line width -NAOP = bytes([9]) # negotiate about output page size -NAOCRD = bytes([10]) # negotiate about CR disposition -NAOHTS = bytes([11]) # negotiate about horizontal tabstops -NAOHTD = bytes([12]) # negotiate about horizontal tab disposition -NAOFFD = bytes([13]) # negotiate about formfeed disposition -NAOVTS = bytes([14]) # negotiate about vertical tab stops -NAOVTD = bytes([15]) # negotiate about vertical tab disposition -NAOLFD = bytes([16]) # negotiate about output LF disposition -XASCII = bytes([17]) # extended ascii character set -LOGOUT = bytes([18]) # force logout -BM = bytes([19]) # byte macro -DET = bytes([20]) # data entry terminal -SUPDUP = bytes([21]) # supdup protocol -SUPDUPOUTPUT = bytes([22]) # supdup output -SNDLOC = bytes([23]) # send location -TTYPE = bytes([24]) # terminal type -EOR = bytes([25]) # end or record -TUID = bytes([26]) # TACACS user identification -OUTMRK = bytes([27]) # output marking -TTYLOC = bytes([28]) # terminal location number -VT3270REGIME = bytes([29]) # 3270 regime -X3PAD = bytes([30]) # X.3 PAD -NAWS = bytes([31]) # window size -TSPEED = bytes([32]) # terminal speed -LFLOW = bytes([33]) # remote flow control -LINEMODE = bytes([34]) # Linemode option -XDISPLOC = bytes([35]) # X Display Location -OLD_ENVIRON = bytes([36]) # Old - Environment variables -AUTHENTICATION = bytes([37]) # Authenticate -ENCRYPT = bytes([38]) # Encryption option -NEW_ENVIRON = bytes([39]) # New - Environment variables -# the following ones come from -# http://www.iana.org/assignments/telnet-options -# Unfortunately, that document does not assign identifiers -# to all of them, so we are making them up -TN3270E = bytes([40]) # TN3270E -XAUTH = bytes([41]) # XAUTH -CHARSET = bytes([42]) # CHARSET -RSP = bytes([43]) # Telnet Remote Serial Port -COM_PORT_OPTION = bytes([44]) # Com Port Control Option -SUPPRESS_LOCAL_ECHO = bytes([45]) # Telnet Suppress Local Echo -TLS = bytes([46]) # Telnet Start TLS -KERMIT = bytes([47]) # KERMIT -SEND_URL = bytes([48]) # SEND-URL -FORWARD_X = bytes([49]) # FORWARD_X -PRAGMA_LOGON = bytes([138]) # TELOPT PRAGMA LOGON -SSPI_LOGON = bytes([139]) # TELOPT SSPI LOGON -PRAGMA_HEARTBEAT = bytes([140]) # TELOPT PRAGMA HEARTBEAT -EXOPL = bytes([255]) # Extended-Options-List -NOOPT = bytes([0]) - - -# poll/select have the advantage of not requiring any extra file descriptor, -# contrarily to epoll/kqueue (also, they require a single syscall). -if hasattr(selectors, 'PollSelector'): - _TelnetSelector = selectors.PollSelector -else: - _TelnetSelector = selectors.SelectSelector - - -class Telnet: - - """Telnet interface class. - - An instance of this class represents a connection to a telnet - server. The instance is initially not connected; the open() - method must be used to establish a connection. Alternatively, the - host name and optional port number can be passed to the - constructor, too. - - Don't try to reopen an already connected instance. - - This class has many read_*() methods. Note that some of them - raise EOFError when the end of the connection is read, because - they can return an empty string for other reasons. See the - individual doc strings. - - read_until(expected, [timeout]) - Read until the expected string has been seen, or a timeout is - hit (default is no timeout); may block. - - read_all() - Read all data until EOF; may block. - - read_some() - Read at least one byte or EOF; may block. - - read_very_eager() - Read all data available already queued or on the socket, - without blocking. - - read_eager() - Read either data already queued or some data available on the - socket, without blocking. - - read_lazy() - Read all data in the raw queue (processing it first), without - doing any socket I/O. - - read_very_lazy() - Reads all data in the cooked queue, without doing any socket - I/O. - - read_sb_data() - Reads available data between SB ... SE sequence. Don't block. - - set_option_negotiation_callback(callback) - Each time a telnet option is read on the input flow, this callback - (if set) is called with the following parameters : - callback(telnet socket, command, option) - option will be chr(0) when there is no option. - No other action is done afterwards by telnetlib. - - """ - - def __init__(self, host=None, port=0, - timeout=socket._GLOBAL_DEFAULT_TIMEOUT): - """Constructor. - - When called without arguments, create an unconnected instance. - With a hostname argument, it connects the instance; port number - and timeout are optional. - """ - self.debuglevel = DEBUGLEVEL - self.host = host - self.port = port - self.timeout = timeout - self.sock = None - self.rawq = b'' - self.irawq = 0 - self.cookedq = b'' - self.eof = 0 - self.iacseq = b'' # Buffer for IAC sequence. - self.sb = 0 # flag for SB and SE sequence. - self.sbdataq = b'' - self.option_callback = None - if host is not None: - self.open(host, port, timeout) - - def open(self, host, port=0, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): - """Connect to a host. - - The optional second argument is the port number, which - defaults to the standard telnet port (23). - - Don't try to reopen an already connected instance. - """ - self.eof = 0 - if not port: - port = TELNET_PORT - self.host = host - self.port = port - self.timeout = timeout - sys.audit("telnetlib.Telnet.open", self, host, port) - self.sock = socket.create_connection((host, port), timeout) - - def __del__(self): - """Destructor -- close the connection.""" - self.close() - - def msg(self, msg, *args): - """Print a debug message, when the debug level is > 0. - - If extra arguments are present, they are substituted in the - message using the standard string formatting operator. - - """ - if self.debuglevel > 0: - print('Telnet(%s,%s):' % (self.host, self.port), end=' ') - if args: - print(msg % args) - else: - print(msg) - - def set_debuglevel(self, debuglevel): - """Set the debug level. - - The higher it is, the more debug output you get (on sys.stdout). - - """ - self.debuglevel = debuglevel - - def close(self): - """Close the connection.""" - sock = self.sock - self.sock = None - self.eof = True - self.iacseq = b'' - self.sb = 0 - if sock: - sock.close() - - def get_socket(self): - """Return the socket object used internally.""" - return self.sock - - def fileno(self): - """Return the fileno() of the socket object used internally.""" - return self.sock.fileno() - - def write(self, buffer): - """Write a string to the socket, doubling any IAC characters. - - Can block if the connection is blocked. May raise - OSError if the connection is closed. - - """ - if IAC in buffer: - buffer = buffer.replace(IAC, IAC+IAC) - sys.audit("telnetlib.Telnet.write", self, buffer) - self.msg("send %r", buffer) - self.sock.sendall(buffer) - - def read_until(self, match, timeout=None): - """Read until a given string is encountered or until timeout. - - When no match is found, return whatever is available instead, - possibly the empty string. Raise EOFError if the connection - is closed and no cooked data is available. - - """ - n = len(match) - self.process_rawq() - i = self.cookedq.find(match) - if i >= 0: - i = i+n - buf = self.cookedq[:i] - self.cookedq = self.cookedq[i:] - return buf - if timeout is not None: - deadline = _time() + timeout - with _TelnetSelector() as selector: - selector.register(self, selectors.EVENT_READ) - while not self.eof: - if selector.select(timeout): - i = max(0, len(self.cookedq)-n) - self.fill_rawq() - self.process_rawq() - i = self.cookedq.find(match, i) - if i >= 0: - i = i+n - buf = self.cookedq[:i] - self.cookedq = self.cookedq[i:] - return buf - if timeout is not None: - timeout = deadline - _time() - if timeout < 0: - break - return self.read_very_lazy() - - def read_all(self): - """Read all data until EOF; block until connection closed.""" - self.process_rawq() - while not self.eof: - self.fill_rawq() - self.process_rawq() - buf = self.cookedq - self.cookedq = b'' - return buf - - def read_some(self): - """Read at least one byte of cooked data unless EOF is hit. - - Return b'' if EOF is hit. Block if no data is immediately - available. - - """ - self.process_rawq() - while not self.cookedq and not self.eof: - self.fill_rawq() - self.process_rawq() - buf = self.cookedq - self.cookedq = b'' - return buf - - def read_very_eager(self): - """Read everything that's possible without blocking in I/O (eager). - - Raise EOFError if connection closed and no cooked data - available. Return b'' if no cooked data available otherwise. - Don't block unless in the midst of an IAC sequence. - - """ - self.process_rawq() - while not self.eof and self.sock_avail(): - self.fill_rawq() - self.process_rawq() - return self.read_very_lazy() - - def read_eager(self): - """Read readily available data. - - Raise EOFError if connection closed and no cooked data - available. Return b'' if no cooked data available otherwise. - Don't block unless in the midst of an IAC sequence. - - """ - self.process_rawq() - while not self.cookedq and not self.eof and self.sock_avail(): - self.fill_rawq() - self.process_rawq() - return self.read_very_lazy() - - def read_lazy(self): - """Process and return data that's already in the queues (lazy). - - Raise EOFError if connection closed and no data available. - Return b'' if no cooked data available otherwise. Don't block - unless in the midst of an IAC sequence. - - """ - self.process_rawq() - return self.read_very_lazy() - - def read_very_lazy(self): - """Return any data available in the cooked queue (very lazy). - - Raise EOFError if connection closed and no data available. - Return b'' if no cooked data available otherwise. Don't block. - - """ - buf = self.cookedq - self.cookedq = b'' - if not buf and self.eof and not self.rawq: - raise EOFError('telnet connection closed') - return buf - - def read_sb_data(self): - """Return any data available in the SB ... SE queue. - - Return b'' if no SB ... SE available. Should only be called - after seeing a SB or SE command. When a new SB command is - found, old unread SB data will be discarded. Don't block. - - """ - buf = self.sbdataq - self.sbdataq = b'' - return buf - - def set_option_negotiation_callback(self, callback): - """Provide a callback function called after each receipt of a telnet option.""" - self.option_callback = callback - - def process_rawq(self): - """Transfer from raw queue to cooked queue. - - Set self.eof when connection is closed. Don't block unless in - the midst of an IAC sequence. - - """ - buf = [b'', b''] - try: - while self.rawq: - c = self.rawq_getchar() - if not self.iacseq: - if c == theNULL: - continue - if c == b"\021": - continue - if c != IAC: - buf[self.sb] = buf[self.sb] + c - continue - else: - self.iacseq += c - elif len(self.iacseq) == 1: - # 'IAC: IAC CMD [OPTION only for WILL/WONT/DO/DONT]' - if c in (DO, DONT, WILL, WONT): - self.iacseq += c - continue - - self.iacseq = b'' - if c == IAC: - buf[self.sb] = buf[self.sb] + c - else: - if c == SB: # SB ... SE start. - self.sb = 1 - self.sbdataq = b'' - elif c == SE: - self.sb = 0 - self.sbdataq = self.sbdataq + buf[1] - buf[1] = b'' - if self.option_callback: - # Callback is supposed to look into - # the sbdataq - self.option_callback(self.sock, c, NOOPT) - else: - # We can't offer automatic processing of - # suboptions. Alas, we should not get any - # unless we did a WILL/DO before. - self.msg('IAC %d not recognized' % ord(c)) - elif len(self.iacseq) == 2: - cmd = self.iacseq[1:2] - self.iacseq = b'' - opt = c - if cmd in (DO, DONT): - self.msg('IAC %s %d', - cmd == DO and 'DO' or 'DONT', ord(opt)) - if self.option_callback: - self.option_callback(self.sock, cmd, opt) - else: - self.sock.sendall(IAC + WONT + opt) - elif cmd in (WILL, WONT): - self.msg('IAC %s %d', - cmd == WILL and 'WILL' or 'WONT', ord(opt)) - if self.option_callback: - self.option_callback(self.sock, cmd, opt) - else: - self.sock.sendall(IAC + DONT + opt) - except EOFError: # raised by self.rawq_getchar() - self.iacseq = b'' # Reset on EOF - self.sb = 0 - self.cookedq = self.cookedq + buf[0] - self.sbdataq = self.sbdataq + buf[1] - - def rawq_getchar(self): - """Get next char from raw queue. - - Block if no data is immediately available. Raise EOFError - when connection is closed. - - """ - if not self.rawq: - self.fill_rawq() - if self.eof: - raise EOFError - c = self.rawq[self.irawq:self.irawq+1] - self.irawq = self.irawq + 1 - if self.irawq >= len(self.rawq): - self.rawq = b'' - self.irawq = 0 - return c - - def fill_rawq(self): - """Fill raw queue from exactly one recv() system call. - - Block if no data is immediately available. Set self.eof when - connection is closed. - - """ - if self.irawq >= len(self.rawq): - self.rawq = b'' - self.irawq = 0 - # The buffer size should be fairly small so as to avoid quadratic - # behavior in process_rawq() above - buf = self.sock.recv(50) - self.msg("recv %r", buf) - self.eof = (not buf) - self.rawq = self.rawq + buf - - def sock_avail(self): - """Test whether data is available on the socket.""" - with _TelnetSelector() as selector: - selector.register(self, selectors.EVENT_READ) - return bool(selector.select(0)) - - def interact(self): - """Interaction function, emulates a very dumb telnet client.""" - if sys.platform == "win32": - self.mt_interact() - return - with _TelnetSelector() as selector: - selector.register(self, selectors.EVENT_READ) - selector.register(sys.stdin, selectors.EVENT_READ) - - while True: - for key, events in selector.select(): - if key.fileobj is self: - try: - text = self.read_eager() - except EOFError: - print('*** Connection closed by remote host ***') - return - if text: - sys.stdout.write(text.decode('ascii')) - sys.stdout.flush() - elif key.fileobj is sys.stdin: - line = sys.stdin.readline().encode('ascii') - if not line: - return - self.write(line) - - def mt_interact(self): - """Multithreaded version of interact().""" - import _thread - _thread.start_new_thread(self.listener, ()) - while 1: - line = sys.stdin.readline() - if not line: - break - self.write(line.encode('ascii')) - - def listener(self): - """Helper for mt_interact() -- this executes in the other thread.""" - while 1: - try: - data = self.read_eager() - except EOFError: - print('*** Connection closed by remote host ***') - return - if data: - sys.stdout.write(data.decode('ascii')) - else: - sys.stdout.flush() - - def expect(self, list, timeout=None): - """Read until one from a list of a regular expressions matches. - - The first argument is a list of regular expressions, either - compiled (re.Pattern instances) or uncompiled (strings). - The optional second argument is a timeout, in seconds; default - is no timeout. - - Return a tuple of three items: the index in the list of the - first regular expression that matches; the re.Match object - returned; and the text read up till and including the match. - - If EOF is read and no text was read, raise EOFError. - Otherwise, when nothing matches, return (-1, None, text) where - text is the text received so far (may be the empty string if a - timeout happened). - - If a regular expression ends with a greedy match (e.g. '.*') - or if more than one expression can match the same input, the - results are undeterministic, and may depend on the I/O timing. - - """ - re = None - list = list[:] - indices = range(len(list)) - for i in indices: - if not hasattr(list[i], "search"): - if not re: import re - list[i] = re.compile(list[i]) - if timeout is not None: - deadline = _time() + timeout - with _TelnetSelector() as selector: - selector.register(self, selectors.EVENT_READ) - while not self.eof: - self.process_rawq() - for i in indices: - m = list[i].search(self.cookedq) - if m: - e = m.end() - text = self.cookedq[:e] - self.cookedq = self.cookedq[e:] - return (i, m, text) - if timeout is not None: - ready = selector.select(timeout) - timeout = deadline - _time() - if not ready: - if timeout < 0: - break - else: - continue - self.fill_rawq() - text = self.read_very_lazy() - if not text and self.eof: - raise EOFError - return (-1, None, text) - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - self.close() - - -def test(): - """Test program for telnetlib. - - Usage: python telnetlib.py [-d] ... [host [port]] - - Default host is localhost; default port is 23. - - """ - debuglevel = 0 - while sys.argv[1:] and sys.argv[1] == '-d': - debuglevel = debuglevel+1 - del sys.argv[1] - host = 'localhost' - if sys.argv[1:]: - host = sys.argv[1] - port = 0 - if sys.argv[2:]: - portstr = sys.argv[2] - try: - port = int(portstr) - except ValueError: - port = socket.getservbyname(portstr, 'tcp') - with Telnet() as tn: - tn.set_debuglevel(debuglevel) - tn.open(host, port, timeout=0.5) - tn.interact() - -if __name__ == '__main__': - test() diff --git a/Lib/test/test_telnetlib.py b/Lib/test/test_telnetlib.py deleted file mode 100644 index a9cade2ee466f9..00000000000000 --- a/Lib/test/test_telnetlib.py +++ /dev/null @@ -1,405 +0,0 @@ -import socket -import selectors -import threading -import contextlib - -from test import support -from test.support import socket_helper, warnings_helper -import unittest - -support.requires_working_socket(module=True) - -telnetlib = warnings_helper.import_deprecated('telnetlib') - -HOST = socket_helper.HOST - -def server(evt, serv): - serv.listen() - evt.set() - try: - conn, addr = serv.accept() - conn.close() - except TimeoutError: - pass - finally: - serv.close() - -class GeneralTests(unittest.TestCase): - - def setUp(self): - self.evt = threading.Event() - self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.sock.settimeout(60) # Safety net. Look issue 11812 - self.port = socket_helper.bind_port(self.sock) - self.thread = threading.Thread(target=server, args=(self.evt,self.sock)) - self.thread.daemon = True - self.thread.start() - self.evt.wait() - - def tearDown(self): - self.thread.join() - del self.thread # Clear out any dangling Thread objects. - - def testBasic(self): - # connects - telnet = telnetlib.Telnet(HOST, self.port) - telnet.sock.close() - - def testContextManager(self): - with telnetlib.Telnet(HOST, self.port) as tn: - self.assertIsNotNone(tn.get_socket()) - self.assertIsNone(tn.get_socket()) - - def testTimeoutDefault(self): - self.assertTrue(socket.getdefaulttimeout() is None) - socket.setdefaulttimeout(30) - try: - telnet = telnetlib.Telnet(HOST, self.port) - finally: - socket.setdefaulttimeout(None) - self.assertEqual(telnet.sock.gettimeout(), 30) - telnet.sock.close() - - def testTimeoutNone(self): - # None, having other default - self.assertTrue(socket.getdefaulttimeout() is None) - socket.setdefaulttimeout(30) - try: - telnet = telnetlib.Telnet(HOST, self.port, timeout=None) - finally: - socket.setdefaulttimeout(None) - self.assertTrue(telnet.sock.gettimeout() is None) - telnet.sock.close() - - def testTimeoutValue(self): - telnet = telnetlib.Telnet(HOST, self.port, timeout=30) - self.assertEqual(telnet.sock.gettimeout(), 30) - telnet.sock.close() - - def testTimeoutOpen(self): - telnet = telnetlib.Telnet() - telnet.open(HOST, self.port, timeout=30) - self.assertEqual(telnet.sock.gettimeout(), 30) - telnet.sock.close() - - def testGetters(self): - # Test telnet getter methods - telnet = telnetlib.Telnet(HOST, self.port, timeout=30) - t_sock = telnet.sock - self.assertEqual(telnet.get_socket(), t_sock) - self.assertEqual(telnet.fileno(), t_sock.fileno()) - telnet.sock.close() - -class SocketStub(object): - ''' a socket proxy that re-defines sendall() ''' - def __init__(self, reads=()): - self.reads = list(reads) # Intentionally make a copy. - self.writes = [] - self.block = False - def sendall(self, data): - self.writes.append(data) - def recv(self, size): - out = b'' - while self.reads and len(out) < size: - out += self.reads.pop(0) - if len(out) > size: - self.reads.insert(0, out[size:]) - out = out[:size] - return out - -class TelnetAlike(telnetlib.Telnet): - def fileno(self): - raise NotImplementedError() - def close(self): pass - def sock_avail(self): - return (not self.sock.block) - def msg(self, msg, *args): - with support.captured_stdout() as out: - telnetlib.Telnet.msg(self, msg, *args) - self._messages += out.getvalue() - return - -class MockSelector(selectors.BaseSelector): - - def __init__(self): - self.keys = {} - - @property - def resolution(self): - return 1e-3 - - def register(self, fileobj, events, data=None): - key = selectors.SelectorKey(fileobj, 0, events, data) - self.keys[fileobj] = key - return key - - def unregister(self, fileobj): - return self.keys.pop(fileobj) - - def select(self, timeout=None): - block = False - for fileobj in self.keys: - if isinstance(fileobj, TelnetAlike): - block = fileobj.sock.block - break - if block: - return [] - else: - return [(key, key.events) for key in self.keys.values()] - - def get_map(self): - return self.keys - - -@contextlib.contextmanager -def test_socket(reads): - def new_conn(*ignored): - return SocketStub(reads) - try: - old_conn = socket.create_connection - socket.create_connection = new_conn - yield None - finally: - socket.create_connection = old_conn - return - -def test_telnet(reads=(), cls=TelnetAlike): - ''' return a telnetlib.Telnet object that uses a SocketStub with - reads queued up to be read ''' - for x in reads: - assert type(x) is bytes, x - with test_socket(reads): - telnet = cls('dummy', 0) - telnet._messages = '' # debuglevel output - return telnet - -class ExpectAndReadTestCase(unittest.TestCase): - def setUp(self): - self.old_selector = telnetlib._TelnetSelector - telnetlib._TelnetSelector = MockSelector - def tearDown(self): - telnetlib._TelnetSelector = self.old_selector - -class ReadTests(ExpectAndReadTestCase): - def test_read_until(self): - """ - read_until(expected, timeout=None) - test the blocking version of read_util - """ - want = [b'xxxmatchyyy'] - telnet = test_telnet(want) - data = telnet.read_until(b'match') - self.assertEqual(data, b'xxxmatch', msg=(telnet.cookedq, telnet.rawq, telnet.sock.reads)) - - reads = [b'x' * 50, b'match', b'y' * 50] - expect = b''.join(reads[:-1]) - telnet = test_telnet(reads) - data = telnet.read_until(b'match') - self.assertEqual(data, expect) - - - def test_read_all(self): - """ - read_all() - Read all data until EOF; may block. - """ - reads = [b'x' * 500, b'y' * 500, b'z' * 500] - expect = b''.join(reads) - telnet = test_telnet(reads) - data = telnet.read_all() - self.assertEqual(data, expect) - return - - def test_read_some(self): - """ - read_some() - Read at least one byte or EOF; may block. - """ - # test 'at least one byte' - telnet = test_telnet([b'x' * 500]) - data = telnet.read_some() - self.assertTrue(len(data) >= 1) - # test EOF - telnet = test_telnet() - data = telnet.read_some() - self.assertEqual(b'', data) - - def _read_eager(self, func_name): - """ - read_*_eager() - Read all data available already queued or on the socket, - without blocking. - """ - want = b'x' * 100 - telnet = test_telnet([want]) - func = getattr(telnet, func_name) - telnet.sock.block = True - self.assertEqual(b'', func()) - telnet.sock.block = False - data = b'' - while True: - try: - data += func() - except EOFError: - break - self.assertEqual(data, want) - - def test_read_eager(self): - # read_eager and read_very_eager make the same guarantees - # (they behave differently but we only test the guarantees) - self._read_eager('read_eager') - self._read_eager('read_very_eager') - # NB -- we need to test the IAC block which is mentioned in the - # docstring but not in the module docs - - def read_very_lazy(self): - want = b'x' * 100 - telnet = test_telnet([want]) - self.assertEqual(b'', telnet.read_very_lazy()) - while telnet.sock.reads: - telnet.fill_rawq() - data = telnet.read_very_lazy() - self.assertEqual(want, data) - self.assertRaises(EOFError, telnet.read_very_lazy) - - def test_read_lazy(self): - want = b'x' * 100 - telnet = test_telnet([want]) - self.assertEqual(b'', telnet.read_lazy()) - data = b'' - while True: - try: - read_data = telnet.read_lazy() - data += read_data - if not read_data: - telnet.fill_rawq() - except EOFError: - break - self.assertTrue(want.startswith(data)) - self.assertEqual(data, want) - -class nego_collector(object): - def __init__(self, sb_getter=None): - self.seen = b'' - self.sb_getter = sb_getter - self.sb_seen = b'' - - def do_nego(self, sock, cmd, opt): - self.seen += cmd + opt - if cmd == tl.SE and self.sb_getter: - sb_data = self.sb_getter() - self.sb_seen += sb_data - -tl = telnetlib - -class WriteTests(unittest.TestCase): - '''The only thing that write does is replace each tl.IAC for - tl.IAC+tl.IAC''' - - def test_write(self): - data_sample = [b'data sample without IAC', - b'data sample with' + tl.IAC + b' one IAC', - b'a few' + tl.IAC + tl.IAC + b' iacs' + tl.IAC, - tl.IAC, - b''] - for data in data_sample: - telnet = test_telnet() - telnet.write(data) - written = b''.join(telnet.sock.writes) - self.assertEqual(data.replace(tl.IAC,tl.IAC+tl.IAC), written) - -class OptionTests(unittest.TestCase): - # RFC 854 commands - cmds = [tl.AO, tl.AYT, tl.BRK, tl.EC, tl.EL, tl.GA, tl.IP, tl.NOP] - - def _test_command(self, data): - """ helper for testing IAC + cmd """ - telnet = test_telnet(data) - data_len = len(b''.join(data)) - nego = nego_collector() - telnet.set_option_negotiation_callback(nego.do_nego) - txt = telnet.read_all() - cmd = nego.seen - self.assertTrue(len(cmd) > 0) # we expect at least one command - self.assertIn(cmd[:1], self.cmds) - self.assertEqual(cmd[1:2], tl.NOOPT) - self.assertEqual(data_len, len(txt + cmd)) - nego.sb_getter = None # break the nego => telnet cycle - - def test_IAC_commands(self): - for cmd in self.cmds: - self._test_command([tl.IAC, cmd]) - self._test_command([b'x' * 100, tl.IAC, cmd, b'y'*100]) - self._test_command([b'x' * 10, tl.IAC, cmd, b'y'*10]) - # all at once - self._test_command([tl.IAC + cmd for (cmd) in self.cmds]) - - def test_SB_commands(self): - # RFC 855, subnegotiations portion - send = [tl.IAC + tl.SB + tl.IAC + tl.SE, - tl.IAC + tl.SB + tl.IAC + tl.IAC + tl.IAC + tl.SE, - tl.IAC + tl.SB + tl.IAC + tl.IAC + b'aa' + tl.IAC + tl.SE, - tl.IAC + tl.SB + b'bb' + tl.IAC + tl.IAC + tl.IAC + tl.SE, - tl.IAC + tl.SB + b'cc' + tl.IAC + tl.IAC + b'dd' + tl.IAC + tl.SE, - ] - telnet = test_telnet(send) - nego = nego_collector(telnet.read_sb_data) - telnet.set_option_negotiation_callback(nego.do_nego) - txt = telnet.read_all() - self.assertEqual(txt, b'') - want_sb_data = tl.IAC + tl.IAC + b'aabb' + tl.IAC + b'cc' + tl.IAC + b'dd' - self.assertEqual(nego.sb_seen, want_sb_data) - self.assertEqual(b'', telnet.read_sb_data()) - nego.sb_getter = None # break the nego => telnet cycle - - def test_debuglevel_reads(self): - # test all the various places that self.msg(...) is called - given_a_expect_b = [ - # Telnet.fill_rawq - (b'a', ": recv b''\n"), - # Telnet.process_rawq - (tl.IAC + bytes([88]), ": IAC 88 not recognized\n"), - (tl.IAC + tl.DO + bytes([1]), ": IAC DO 1\n"), - (tl.IAC + tl.DONT + bytes([1]), ": IAC DONT 1\n"), - (tl.IAC + tl.WILL + bytes([1]), ": IAC WILL 1\n"), - (tl.IAC + tl.WONT + bytes([1]), ": IAC WONT 1\n"), - ] - for a, b in given_a_expect_b: - telnet = test_telnet([a]) - telnet.set_debuglevel(1) - txt = telnet.read_all() - self.assertIn(b, telnet._messages) - return - - def test_debuglevel_write(self): - telnet = test_telnet() - telnet.set_debuglevel(1) - telnet.write(b'xxx') - expected = "send b'xxx'\n" - self.assertIn(expected, telnet._messages) - - def test_debug_accepts_str_port(self): - # Issue 10695 - with test_socket([]): - telnet = TelnetAlike('dummy', '0') - telnet._messages = '' - telnet.set_debuglevel(1) - telnet.msg('test') - self.assertRegex(telnet._messages, r'0.*test') - - -class ExpectTests(ExpectAndReadTestCase): - def test_expect(self): - """ - expect(expected, [timeout]) - Read until the expected string has been seen, or a timeout is - hit (default is no timeout); may block. - """ - want = [b'x' * 10, b'match', b'y' * 10] - telnet = test_telnet(want) - (_,_,data) = telnet.expect([b'match']) - self.assertEqual(data, b''.join(want[:-1])) - - -if __name__ == '__main__': - unittest.main() diff --git a/Misc/NEWS.d/3.9.0a1.rst b/Misc/NEWS.d/3.9.0a1.rst index 0888a5c43087b5..7da4385973185a 100644 --- a/Misc/NEWS.d/3.9.0a1.rst +++ b/Misc/NEWS.d/3.9.0a1.rst @@ -99,7 +99,7 @@ after whitespace, e.g. '127.0.0.1 whatever'. Adds audit events for :mod:`ensurepip`, :mod:`ftplib`, :mod:`glob`, :mod:`imaplib`, :mod:`nntplib`, :mod:`pdb`, :mod:`poplib`, :mod:`shutil`, -:mod:`smtplib`, :mod:`sqlite3`, :mod:`subprocess`, :mod:`telnetlib`, +:mod:`smtplib`, :mod:`sqlite3`, :mod:`subprocess`, :mod:`!telnetlib`, :mod:`tempfile` and :mod:`webbrowser`, as well as :func:`os.listdir`, :func:`os.scandir` and :func:`breakpoint`. diff --git a/Misc/NEWS.d/next/Library/2023-05-23-02-20-13.gh-issue-104773.7K59zr.rst b/Misc/NEWS.d/next/Library/2023-05-23-02-20-13.gh-issue-104773.7K59zr.rst new file mode 100644 index 00000000000000..0f3162ef509a82 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-23-02-20-13.gh-issue-104773.7K59zr.rst @@ -0,0 +1,2 @@ +:pep:`594`: Remove the :mod:`!telnetlib` module, deprecated in Python 3.11. +Patch by Victor Stinner. diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index ed4a0ac2dd32de..4b0580557fb7a4 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -261,7 +261,6 @@ static const char* _Py_stdlib_module_names[] = { "syslog", "tabnanny", "tarfile", -"telnetlib", "tempfile", "termios", "textwrap", diff --git a/Tools/wasm/wasm_assets.py b/Tools/wasm/wasm_assets.py index 1fc97fd5e70a10..22bf9eabdcd6b4 100755 --- a/Tools/wasm/wasm_assets.py +++ b/Tools/wasm/wasm_assets.py @@ -78,7 +78,6 @@ "poplib.py", "smtplib.py", "socketserver.py", - "telnetlib.py", # keep urllib.parse for pydoc "urllib/error.py", "urllib/request.py", From abdda5b13388aa21923f6ce104242adff518d430 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Tue, 23 May 2023 15:44:26 +0200 Subject: [PATCH 36/47] gh-92871: Remove typing.{io,re} namespaces (#92873) Closes #92871 Co-authored-by: Alex Waygood Co-authored-by: Jelle Zijlstra --- Doc/library/typing.rst | 11 ----- Doc/whatsnew/3.13.rst | 3 ++ Lib/test/test_typing.py | 24 +---------- Lib/typing.py | 43 +------------------ ...2-05-17-10-46-44.gh-issue-92871.GVogrT.rst | 2 + 5 files changed, 7 insertions(+), 76 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-05-17-10-46-44.gh-issue-92871.GVogrT.rst diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index c90cb411acde07..92943c46ef5132 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2074,10 +2074,6 @@ Other concrete types represent the types of I/O streams such as returned by :func:`open`. - .. deprecated-removed:: 3.8 3.13 - The ``typing.io`` namespace is deprecated and will be removed. - These types should be directly imported from ``typing`` instead. - .. class:: Pattern Match @@ -2088,10 +2084,6 @@ Other concrete types ``Pattern[str]``, ``Pattern[bytes]``, ``Match[str]``, or ``Match[bytes]``. - .. deprecated-removed:: 3.8 3.13 - The ``typing.re`` namespace is deprecated and will be removed. - These types should be directly imported from ``typing`` instead. - .. deprecated:: 3.9 Classes ``Pattern`` and ``Match`` from :mod:`re` now support ``[]``. See :pep:`585` and :ref:`types-genericalias`. @@ -2981,9 +2973,6 @@ convenience. This is subject to change, and not all deprecations are listed. +----------------------------------+---------------+-------------------+----------------+ | Feature | Deprecated in | Projected removal | PEP/issue | +==================================+===============+===================+================+ -| ``typing.io`` and ``typing.re`` | 3.8 | 3.13 | :issue:`38291` | -| submodules | | | | -+----------------------------------+---------------+-------------------+----------------+ | ``typing`` versions of standard | 3.9 | Undecided | :pep:`585` | | collections | | | | +----------------------------------+---------------+-------------------+----------------+ diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 8256f429183a3a..602a865b5930e8 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -107,6 +107,9 @@ Removed `Exscript `_ instead. (Contributed by Victor Stinner in :gh:`104773`.) +* Namespaces ``typing.io`` and ``typing.re``, deprecated in Python 3.8, + are now removed. The items in those namespaces can be imported directly + from :mod:`typing`. (Contributed by Sebastian Rittau in :gh:`92871`.) Porting to Python 3.13 ====================== diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 9a3e64289ee877..b4a5a68a2f7e4e 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -7529,17 +7529,6 @@ def stuff(a: BinaryIO) -> bytes: a = stuff.__annotations__['a'] self.assertEqual(a.__parameters__, ()) - def test_io_submodule(self): - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings("default", category=DeprecationWarning) - from typing.io import IO, TextIO, BinaryIO, __all__, __name__ - self.assertIs(IO, typing.IO) - self.assertIs(TextIO, typing.TextIO) - self.assertIs(BinaryIO, typing.BinaryIO) - self.assertEqual(set(__all__), set(['IO', 'TextIO', 'BinaryIO'])) - self.assertEqual(__name__, 'typing.io') - self.assertEqual(len(w), 1) - class RETests(BaseTestCase): # Much of this is really testing _TypeAlias. @@ -7584,16 +7573,6 @@ def test_repr(self): self.assertEqual(repr(Match[str]), 'typing.Match[str]') self.assertEqual(repr(Match[bytes]), 'typing.Match[bytes]') - def test_re_submodule(self): - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings("default", category=DeprecationWarning) - from typing.re import Match, Pattern, __all__, __name__ - self.assertIs(Match, typing.Match) - self.assertIs(Pattern, typing.Pattern) - self.assertEqual(set(__all__), set(['Match', 'Pattern'])) - self.assertEqual(__name__, 'typing.re') - self.assertEqual(len(w), 1) - def test_cannot_subclass(self): with self.assertRaisesRegex( TypeError, @@ -8765,7 +8744,7 @@ def test_all(self): # Context managers. self.assertIn('ContextManager', a) self.assertIn('AsyncContextManager', a) - # Check that io and re are not exported. + # Check that former namespaces io and re are not exported. self.assertNotIn('io', a) self.assertNotIn('re', a) # Spot-check that stdlib modules aren't exported. @@ -8785,7 +8764,6 @@ def test_all_exported_names(self): if k in actual_all or ( # avoid private names not k.startswith('_') and - k not in {'io', 're'} and # there's a few types and metaclasses that aren't exported not k.endswith(('Meta', '_contra', '_co')) and not k.upper() == k and diff --git a/Lib/typing.py b/Lib/typing.py index 96393d6a02815b..95dbc0b85bcddf 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -16,7 +16,6 @@ no_type_check_decorator. * Generic aliases for collections.abc ABCs and few additional protocols. * Special types: NewType, NamedTuple, TypedDict. -* Wrapper submodules for re and io related types. """ from abc import abstractmethod, ABCMeta @@ -27,7 +26,7 @@ import contextlib import functools import operator -import re as stdlib_re # Avoid confusion with the re we export. +import re as stdlib_re # Avoid confusion with the typing.re namespace on <=3.11 import sys import types import warnings @@ -158,10 +157,6 @@ 'Unpack', ] -# The pseudo-submodules 're' and 'io' are part of the public -# namespace, but excluded from __all__ because they might stomp on -# legitimate imports of those modules. - def _type_convert(arg, module=None, *, allow_special_forms=False): """For converting None to type(None), and strings to ForwardRef.""" @@ -3150,45 +3145,9 @@ def __enter__(self) -> 'TextIO': pass -class _DeprecatedType(type): - def __getattribute__(cls, name): - if name not in ("__dict__", "__module__") and name in cls.__dict__: - warnings.warn( - f"{cls.__name__} is deprecated, import directly " - f"from typing instead. {cls.__name__} will be removed " - "in Python 3.12.", - DeprecationWarning, - stacklevel=2, - ) - return super().__getattribute__(name) - - -class io(metaclass=_DeprecatedType): - """Wrapper namespace for IO generic classes.""" - - __all__ = ['IO', 'TextIO', 'BinaryIO'] - IO = IO - TextIO = TextIO - BinaryIO = BinaryIO - - -io.__name__ = __name__ + '.io' -sys.modules[io.__name__] = io - Pattern = _alias(stdlib_re.Pattern, 1) Match = _alias(stdlib_re.Match, 1) -class re(metaclass=_DeprecatedType): - """Wrapper namespace for re type aliases.""" - - __all__ = ['Pattern', 'Match'] - Pattern = Pattern - Match = Match - - -re.__name__ = __name__ + '.re' -sys.modules[re.__name__] = re - def reveal_type[T](obj: T, /) -> T: """Reveal the inferred type of a variable. diff --git a/Misc/NEWS.d/next/Library/2022-05-17-10-46-44.gh-issue-92871.GVogrT.rst b/Misc/NEWS.d/next/Library/2022-05-17-10-46-44.gh-issue-92871.GVogrT.rst new file mode 100644 index 00000000000000..992f8afadbe912 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-05-17-10-46-44.gh-issue-92871.GVogrT.rst @@ -0,0 +1,2 @@ +Remove the ``typing.io`` and ``typing.re`` namespaces, deprecated since Python +3.8. All items are still available from the main :mod:`typing` module. From 76170f545870db93b6c45cae09ff34b18639ee97 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Tue, 23 May 2023 22:52:36 +0900 Subject: [PATCH 37/47] gh-104469: Convert _testcapi/long to use AC (gh-104720) --- Modules/_testcapi/clinic/long.c.h | 166 ++++++++++++++++++++++++++++++ Modules/_testcapi/long.c | 119 ++++++++++++++------- 2 files changed, 250 insertions(+), 35 deletions(-) create mode 100644 Modules/_testcapi/clinic/long.c.h diff --git a/Modules/_testcapi/clinic/long.c.h b/Modules/_testcapi/clinic/long.c.h new file mode 100644 index 00000000000000..95885e0ae17a6c --- /dev/null +++ b/Modules/_testcapi/clinic/long.c.h @@ -0,0 +1,166 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif + + +PyDoc_STRVAR(_testcapi_test_long_api__doc__, +"test_long_api($module, /)\n" +"--\n" +"\n"); + +#define _TESTCAPI_TEST_LONG_API_METHODDEF \ + {"test_long_api", (PyCFunction)_testcapi_test_long_api, METH_NOARGS, _testcapi_test_long_api__doc__}, + +static PyObject * +_testcapi_test_long_api_impl(PyObject *module); + +static PyObject * +_testcapi_test_long_api(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testcapi_test_long_api_impl(module); +} + +PyDoc_STRVAR(_testcapi_test_longlong_api__doc__, +"test_longlong_api($module, /)\n" +"--\n" +"\n"); + +#define _TESTCAPI_TEST_LONGLONG_API_METHODDEF \ + {"test_longlong_api", (PyCFunction)_testcapi_test_longlong_api, METH_NOARGS, _testcapi_test_longlong_api__doc__}, + +static PyObject * +_testcapi_test_longlong_api_impl(PyObject *module); + +static PyObject * +_testcapi_test_longlong_api(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testcapi_test_longlong_api_impl(module); +} + +PyDoc_STRVAR(_testcapi_test_long_and_overflow__doc__, +"test_long_and_overflow($module, /)\n" +"--\n" +"\n" +"Test the PyLong_AsLongAndOverflow API.\n" +"\n" +"General conversion to PY_LONG is tested by test_long_api_inner.\n" +"This test will concentrate on proper handling of overflow."); + +#define _TESTCAPI_TEST_LONG_AND_OVERFLOW_METHODDEF \ + {"test_long_and_overflow", (PyCFunction)_testcapi_test_long_and_overflow, METH_NOARGS, _testcapi_test_long_and_overflow__doc__}, + +static PyObject * +_testcapi_test_long_and_overflow_impl(PyObject *module); + +static PyObject * +_testcapi_test_long_and_overflow(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testcapi_test_long_and_overflow_impl(module); +} + +PyDoc_STRVAR(_testcapi_test_long_long_and_overflow__doc__, +"test_long_long_and_overflow($module, /)\n" +"--\n" +"\n" +"Test the PyLong_AsLongLongAndOverflow API.\n" +"\n" +"General conversion to long long is tested by test_long_api_inner.\n" +"This test will concentrate on proper handling of overflow."); + +#define _TESTCAPI_TEST_LONG_LONG_AND_OVERFLOW_METHODDEF \ + {"test_long_long_and_overflow", (PyCFunction)_testcapi_test_long_long_and_overflow, METH_NOARGS, _testcapi_test_long_long_and_overflow__doc__}, + +static PyObject * +_testcapi_test_long_long_and_overflow_impl(PyObject *module); + +static PyObject * +_testcapi_test_long_long_and_overflow(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testcapi_test_long_long_and_overflow_impl(module); +} + +PyDoc_STRVAR(_testcapi_test_long_as_size_t__doc__, +"test_long_as_size_t($module, /)\n" +"--\n" +"\n" +"Test the PyLong_As{Size,Ssize}_t API.\n" +"\n" +"At present this just tests that non-integer arguments are handled correctly.\n" +"It should be extended to test overflow handling."); + +#define _TESTCAPI_TEST_LONG_AS_SIZE_T_METHODDEF \ + {"test_long_as_size_t", (PyCFunction)_testcapi_test_long_as_size_t, METH_NOARGS, _testcapi_test_long_as_size_t__doc__}, + +static PyObject * +_testcapi_test_long_as_size_t_impl(PyObject *module); + +static PyObject * +_testcapi_test_long_as_size_t(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testcapi_test_long_as_size_t_impl(module); +} + +PyDoc_STRVAR(_testcapi_test_long_as_unsigned_long_long_mask__doc__, +"test_long_as_unsigned_long_long_mask($module, /)\n" +"--\n" +"\n"); + +#define _TESTCAPI_TEST_LONG_AS_UNSIGNED_LONG_LONG_MASK_METHODDEF \ + {"test_long_as_unsigned_long_long_mask", (PyCFunction)_testcapi_test_long_as_unsigned_long_long_mask, METH_NOARGS, _testcapi_test_long_as_unsigned_long_long_mask__doc__}, + +static PyObject * +_testcapi_test_long_as_unsigned_long_long_mask_impl(PyObject *module); + +static PyObject * +_testcapi_test_long_as_unsigned_long_long_mask(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testcapi_test_long_as_unsigned_long_long_mask_impl(module); +} + +PyDoc_STRVAR(_testcapi_test_long_as_double__doc__, +"test_long_as_double($module, /)\n" +"--\n" +"\n"); + +#define _TESTCAPI_TEST_LONG_AS_DOUBLE_METHODDEF \ + {"test_long_as_double", (PyCFunction)_testcapi_test_long_as_double, METH_NOARGS, _testcapi_test_long_as_double__doc__}, + +static PyObject * +_testcapi_test_long_as_double_impl(PyObject *module); + +static PyObject * +_testcapi_test_long_as_double(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testcapi_test_long_as_double_impl(module); +} + +PyDoc_STRVAR(_testcapi_test_long_numbits__doc__, +"test_long_numbits($module, /)\n" +"--\n" +"\n"); + +#define _TESTCAPI_TEST_LONG_NUMBITS_METHODDEF \ + {"test_long_numbits", (PyCFunction)_testcapi_test_long_numbits, METH_NOARGS, _testcapi_test_long_numbits__doc__}, + +static PyObject * +_testcapi_test_long_numbits_impl(PyObject *module); + +static PyObject * +_testcapi_test_long_numbits(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testcapi_test_long_numbits_impl(module); +} + +PyDoc_STRVAR(_testcapi_call_long_compact_api__doc__, +"call_long_compact_api($module, arg, /)\n" +"--\n" +"\n"); + +#define _TESTCAPI_CALL_LONG_COMPACT_API_METHODDEF \ + {"call_long_compact_api", (PyCFunction)_testcapi_call_long_compact_api, METH_O, _testcapi_call_long_compact_api__doc__}, +/*[clinic end generated code: output=d000a1b58fa81eab input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 61dd96596dad1c..ede43f60d06f95 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -1,4 +1,10 @@ #include "parts.h" +#include "clinic/long.c.h" + +/*[clinic input] +module _testcapi +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=6361033e795369fc]*/ static PyObject * @@ -40,8 +46,13 @@ raise_test_long_error(const char* msg) #include "testcapi_long.h" +/*[clinic input] +_testcapi.test_long_api +[clinic start generated code]*/ + static PyObject * -test_long_api(PyObject* self, PyObject *Py_UNUSED(ignored)) +_testcapi_test_long_api_impl(PyObject *module) +/*[clinic end generated code: output=4405798ca1e9f444 input=e9b8880d7331c688]*/ { return TESTNAME(raise_test_long_error); } @@ -68,8 +79,13 @@ raise_test_longlong_error(const char* msg) #include "testcapi_long.h" +/*[clinic input] +_testcapi.test_longlong_api +[clinic start generated code]*/ + static PyObject * -test_longlong_api(PyObject* self, PyObject *args) +_testcapi_test_longlong_api_impl(PyObject *module) +/*[clinic end generated code: output=2b3414ba8c31dfe6 input=ccbb2a48c2b3c4a5]*/ { return TESTNAME(raise_test_longlong_error); } @@ -81,13 +97,19 @@ test_longlong_api(PyObject* self, PyObject *args) #undef F_U_TO_PY #undef F_PY_TO_U -/* Test the PyLong_AsLongAndOverflow API. General conversion to PY_LONG - is tested by test_long_api_inner. This test will concentrate on proper - handling of overflow. -*/ + +/*[clinic input] +_testcapi.test_long_and_overflow + +Test the PyLong_AsLongAndOverflow API. + +General conversion to PY_LONG is tested by test_long_api_inner. +This test will concentrate on proper handling of overflow. +[clinic start generated code]*/ static PyObject * -test_long_and_overflow(PyObject *self, PyObject *Py_UNUSED(ignored)) +_testcapi_test_long_and_overflow_impl(PyObject *module) +/*[clinic end generated code: output=f8460ca115e31d8e input=762f6b62da0a3cdc]*/ { PyObject *num, *one, *temp; long value; @@ -243,13 +265,18 @@ test_long_and_overflow(PyObject *self, PyObject *Py_UNUSED(ignored)) Py_RETURN_NONE; } -/* Test the PyLong_AsLongLongAndOverflow API. General conversion to - long long is tested by test_long_api_inner. This test will - concentrate on proper handling of overflow. -*/ +/*[clinic input] +_testcapi.test_long_long_and_overflow + +Test the PyLong_AsLongLongAndOverflow API. + +General conversion to long long is tested by test_long_api_inner. +This test will concentrate on proper handling of overflow. +[clinic start generated code]*/ static PyObject * -test_long_long_and_overflow(PyObject *self, PyObject *Py_UNUSED(ignored)) +_testcapi_test_long_long_and_overflow_impl(PyObject *module) +/*[clinic end generated code: output=0b92330786f45483 input=544bb0aefe5e8a9e]*/ { PyObject *num, *one, *temp; long long value; @@ -405,13 +432,18 @@ test_long_long_and_overflow(PyObject *self, PyObject *Py_UNUSED(ignored)) Py_RETURN_NONE; } -/* Test the PyLong_As{Size,Ssize}_t API. At present this just tests that - non-integer arguments are handled correctly. It should be extended to - test overflow handling. - */ +/*[clinic input] +_testcapi.test_long_as_size_t + +Test the PyLong_As{Size,Ssize}_t API. + +At present this just tests that non-integer arguments are handled correctly. +It should be extended to test overflow handling. +[clinic start generated code]*/ static PyObject * -test_long_as_size_t(PyObject *self, PyObject *Py_UNUSED(ignored)) +_testcapi_test_long_as_size_t_impl(PyObject *module) +/*[clinic end generated code: output=f6490ea2b41e6173 input=922990c4a3edfb0d]*/ { size_t out_u; Py_ssize_t out_s; @@ -442,9 +474,13 @@ test_long_as_size_t(PyObject *self, PyObject *Py_UNUSED(ignored)) return Py_None; } +/*[clinic input] +_testcapi.test_long_as_unsigned_long_long_mask +[clinic start generated code]*/ + static PyObject * -test_long_as_unsigned_long_long_mask(PyObject *self, - PyObject *Py_UNUSED(ignored)) +_testcapi_test_long_as_unsigned_long_long_mask_impl(PyObject *module) +/*[clinic end generated code: output=e3e16cd0189440cc input=eb2438493ae7b9af]*/ { unsigned long long res = PyLong_AsUnsignedLongLongMask(NULL); @@ -462,12 +498,13 @@ test_long_as_unsigned_long_long_mask(PyObject *self, Py_RETURN_NONE; } -/* Test the PyLong_AsDouble API. At present this just tests that - non-integer arguments are handled correctly. - */ +/*[clinic input] +_testcapi.test_long_as_double +[clinic start generated code]*/ static PyObject * -test_long_as_double(PyObject *self, PyObject *Py_UNUSED(ignored)) +_testcapi_test_long_as_double_impl(PyObject *module) +/*[clinic end generated code: output=deca0898e15adde5 input=c77bc88ef5a1de76]*/ { double out; @@ -487,9 +524,13 @@ test_long_as_double(PyObject *self, PyObject *Py_UNUSED(ignored)) return Py_None; } -/* Simple test of _PyLong_NumBits and _PyLong_Sign. */ +/*[clinic input] +_testcapi.test_long_numbits +[clinic start generated code]*/ + static PyObject * -test_long_numbits(PyObject *self, PyObject *Py_UNUSED(ignored)) +_testcapi_test_long_numbits_impl(PyObject *module) +/*[clinic end generated code: output=9eaf8458cb15d7f7 input=265c02d48a13059e]*/ { struct triple { long input; @@ -534,8 +575,16 @@ test_long_numbits(PyObject *self, PyObject *Py_UNUSED(ignored)) Py_RETURN_NONE; } +/*[clinic input] +_testcapi.call_long_compact_api + arg: object + / +[clinic start generated code]*/ + static PyObject * -check_long_compact_api(PyObject *self, PyObject *arg) +_testcapi_call_long_compact_api(PyObject *module, PyObject *arg) +/*[clinic end generated code: output=7e3894f611b1b2b7 input=87b87396967af14c]*/ + { assert(PyLong_Check(arg)); int is_compact = PyUnstable_Long_IsCompact((PyLongObject*)arg); @@ -547,15 +596,15 @@ check_long_compact_api(PyObject *self, PyObject *arg) } static PyMethodDef test_methods[] = { - {"test_long_and_overflow", test_long_and_overflow, METH_NOARGS}, - {"test_long_api", test_long_api, METH_NOARGS}, - {"test_long_as_double", test_long_as_double, METH_NOARGS}, - {"test_long_as_size_t", test_long_as_size_t, METH_NOARGS}, - {"test_long_as_unsigned_long_long_mask", test_long_as_unsigned_long_long_mask, METH_NOARGS}, - {"test_long_long_and_overflow",test_long_long_and_overflow, METH_NOARGS}, - {"test_long_numbits", test_long_numbits, METH_NOARGS}, - {"test_longlong_api", test_longlong_api, METH_NOARGS}, - {"call_long_compact_api", check_long_compact_api, METH_O}, + _TESTCAPI_TEST_LONG_AND_OVERFLOW_METHODDEF + _TESTCAPI_TEST_LONG_API_METHODDEF + _TESTCAPI_TEST_LONG_AS_DOUBLE_METHODDEF + _TESTCAPI_TEST_LONG_AS_SIZE_T_METHODDEF + _TESTCAPI_TEST_LONG_AS_UNSIGNED_LONG_LONG_MASK_METHODDEF + _TESTCAPI_TEST_LONG_LONG_AND_OVERFLOW_METHODDEF + _TESTCAPI_TEST_LONG_NUMBITS_METHODDEF + _TESTCAPI_TEST_LONGLONG_API_METHODDEF + _TESTCAPI_CALL_LONG_COMPACT_API_METHODDEF {NULL}, }; From ddb14859535ab8091381b9d0baf32dbe245b5e65 Mon Sep 17 00:00:00 2001 From: Daniel Fortunov Date: Tue, 23 May 2023 15:11:29 +0100 Subject: [PATCH 38/47] gh-81005: Refactor str tests to reflect that str and unicode are merged in Python 3 (#13172) Co-authored-by: Hugo van Kemenade --- Lib/test/string_tests.py | 26 +++++-------- Lib/test/test_builtin.py | 2 +- Lib/test/{test_unicode.py => test_str.py} | 37 +++++++++---------- Lib/test/test_userstring.py | 3 +- ...3-05-19-08-06-06.gh-issue-81005.-q7m9W.rst | 2 + 5 files changed, 31 insertions(+), 39 deletions(-) rename Lib/test/{test_unicode.py => test_str.py} (99%) create mode 100644 Misc/NEWS.d/next/Tests/2023-05-19-08-06-06.gh-issue-81005.-q7m9W.rst diff --git a/Lib/test/string_tests.py b/Lib/test/string_tests.py index 709cac7a27a449..a6ea2f378b37ac 100644 --- a/Lib/test/string_tests.py +++ b/Lib/test/string_tests.py @@ -8,18 +8,12 @@ from collections import UserList import random + class Sequence: def __init__(self, seq='wxyz'): self.seq = seq def __len__(self): return len(self.seq) def __getitem__(self, i): return self.seq[i] -class BadSeq1(Sequence): - def __init__(self): self.seq = [7, 'hello', 123] - def __str__(self): return '{0} {1} {2}'.format(*self.seq) - -class BadSeq2(Sequence): - def __init__(self): self.seq = ['a', 'b', 'c'] - def __len__(self): return 8 class BaseTest: # These tests are for buffers of values (bytes) and not @@ -27,7 +21,7 @@ class BaseTest: # and various string implementations # The type to be tested - # Change in subclasses to change the behaviour of fixtesttype() + # Change in subclasses to change the behaviour of fixtype() type2test = None # Whether the "contained items" of the container are integers in @@ -36,7 +30,7 @@ class BaseTest: contains_bytes = False # All tests pass their arguments to the testing methods - # as str objects. fixtesttype() can be used to propagate + # as str objects. fixtype() can be used to propagate # these arguments to the appropriate type def fixtype(self, obj): if isinstance(obj, str): @@ -1096,7 +1090,7 @@ def test_splitlines(self): self.checkraises(TypeError, 'abc', 'splitlines', 42, 42) -class CommonTest(BaseTest): +class StringLikeTest(BaseTest): # This testcase contains tests that can be used in all # stringlike classes. Currently this is str and UserString. @@ -1127,11 +1121,6 @@ def test_capitalize_nonascii(self): self.checkequal('\u019b\u1d00\u1d86\u0221\u1fb7', '\u019b\u1d00\u1d86\u0221\u1fb7', 'capitalize') - -class MixinStrUnicodeUserStringTest: - # additional tests that only work for - # stringlike objects, i.e. str, UserString - def test_startswith(self): self.checkequal(True, 'hello', 'startswith', 'he') self.checkequal(True, 'hello', 'startswith', 'hello') @@ -1313,8 +1302,11 @@ def test_join(self): self.checkequal(((('a' * i) + '-') * i)[:-1], '-', 'join', ('a' * i,) * i) - #self.checkequal(str(BadSeq1()), ' ', 'join', BadSeq1()) - self.checkequal('a b c', ' ', 'join', BadSeq2()) + class LiesAboutLengthSeq(Sequence): + def __init__(self): self.seq = ['a', 'b', 'c'] + def __len__(self): return 8 + + self.checkequal('a b c', ' ', 'join', LiesAboutLengthSeq()) self.checkraises(TypeError, ' ', 'join') self.checkraises(TypeError, ' ', 'join', None) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 1257b529038afb..f5a5c037f1bf1b 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1614,7 +1614,7 @@ def test_setattr(self): msg = r"^attribute name must be string, not 'int'$" self.assertRaisesRegex(TypeError, msg, setattr, sys, 1, 'spam') - # test_str(): see test_unicode.py and test_bytes.py for str() tests. + # test_str(): see test_str.py and test_bytes.py for str() tests. def test_sum(self): self.assertEqual(sum([]), 0) diff --git a/Lib/test/test_unicode.py b/Lib/test/test_str.py similarity index 99% rename from Lib/test/test_unicode.py rename to Lib/test/test_str.py index 4ebbb9d32a3d75..a6bcc2455de15e 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_str.py @@ -55,8 +55,7 @@ def duplicate_string(text): class StrSubclass(str): pass -class UnicodeTest(string_tests.CommonTest, - string_tests.MixinStrUnicodeUserStringTest, +class StrTest(string_tests.StringLikeTest, string_tests.MixinStrUnicodeTest, unittest.TestCase): @@ -213,7 +212,7 @@ def test_pickle_iterator(self): self.assertEqual(case, pickled) def test_count(self): - string_tests.CommonTest.test_count(self) + string_tests.StringLikeTest.test_count(self) # check mixed argument types self.checkequalnofix(3, 'aaa', 'count', 'a') self.checkequalnofix(0, 'aaa', 'count', 'b') @@ -243,7 +242,7 @@ class MyStr(str): self.checkequal(3, MyStr('aaa'), 'count', 'a') def test_find(self): - string_tests.CommonTest.test_find(self) + string_tests.StringLikeTest.test_find(self) # test implementation details of the memchr fast path self.checkequal(100, 'a' * 100 + '\u0102', 'find', '\u0102') self.checkequal(-1, 'a' * 100 + '\u0102', 'find', '\u0201') @@ -288,7 +287,7 @@ def test_find(self): self.checkequal(-1, '\u0102' * 100, 'find', '\u0102\U00100304') def test_rfind(self): - string_tests.CommonTest.test_rfind(self) + string_tests.StringLikeTest.test_rfind(self) # test implementation details of the memrchr fast path self.checkequal(0, '\u0102' + 'a' * 100 , 'rfind', '\u0102') self.checkequal(-1, '\u0102' + 'a' * 100 , 'rfind', '\u0201') @@ -329,7 +328,7 @@ def test_rfind(self): self.checkequal(-1, '\u0102' * 100, 'rfind', '\U00100304\u0102') def test_index(self): - string_tests.CommonTest.test_index(self) + string_tests.StringLikeTest.test_index(self) self.checkequalnofix(0, 'abcdefghiabc', 'index', '') self.checkequalnofix(3, 'abcdefghiabc', 'index', 'def') self.checkequalnofix(0, 'abcdefghiabc', 'index', 'abc') @@ -353,7 +352,7 @@ def test_index(self): self.assertRaises(ValueError, ('\u0102' * 100).index, '\u0102\U00100304') def test_rindex(self): - string_tests.CommonTest.test_rindex(self) + string_tests.StringLikeTest.test_rindex(self) self.checkequalnofix(12, 'abcdefghiabc', 'rindex', '') self.checkequalnofix(3, 'abcdefghiabc', 'rindex', 'def') self.checkequalnofix(9, 'abcdefghiabc', 'rindex', 'abc') @@ -449,7 +448,7 @@ def test_maketrans_translate(self): self.assertRaises(TypeError, 'abababc'.translate, 'abc', 'xyz') def test_split(self): - string_tests.CommonTest.test_split(self) + string_tests.StringLikeTest.test_split(self) # test mixed kinds for left, right in ('ba', '\u0101\u0100', '\U00010301\U00010300'): @@ -466,7 +465,7 @@ def test_split(self): left + delim * 2 + right, 'split', delim *2) def test_rsplit(self): - string_tests.CommonTest.test_rsplit(self) + string_tests.StringLikeTest.test_rsplit(self) # test mixed kinds for left, right in ('ba', 'юё', '\u0101\u0100', '\U00010301\U00010300'): left *= 9 @@ -486,7 +485,7 @@ def test_rsplit(self): left + right, 'rsplit', None) def test_partition(self): - string_tests.MixinStrUnicodeUserStringTest.test_partition(self) + string_tests.StringLikeTest.test_partition(self) # test mixed kinds self.checkequal(('ABCDEFGH', '', ''), 'ABCDEFGH', 'partition', '\u4200') for left, right in ('ba', '\u0101\u0100', '\U00010301\U00010300'): @@ -503,7 +502,7 @@ def test_partition(self): left + delim * 2 + right, 'partition', delim * 2) def test_rpartition(self): - string_tests.MixinStrUnicodeUserStringTest.test_rpartition(self) + string_tests.StringLikeTest.test_rpartition(self) # test mixed kinds self.checkequal(('', '', 'ABCDEFGH'), 'ABCDEFGH', 'rpartition', '\u4200') for left, right in ('ba', '\u0101\u0100', '\U00010301\U00010300'): @@ -520,7 +519,7 @@ def test_rpartition(self): left + delim * 2 + right, 'rpartition', delim * 2) def test_join(self): - string_tests.MixinStrUnicodeUserStringTest.test_join(self) + string_tests.StringLikeTest.test_join(self) class MyWrapper: def __init__(self, sval): self.sval = sval @@ -547,7 +546,7 @@ def test_join_overflow(self): self.assertRaises(OverflowError, ''.join, seq) def test_replace(self): - string_tests.CommonTest.test_replace(self) + string_tests.StringLikeTest.test_replace(self) # method call forwarded from str implementation because of unicode argument self.checkequalnofix('one@two!three!', 'one!two!three!', 'replace', '!', '@', 1) @@ -866,7 +865,7 @@ def test_surrogates(self): def test_lower(self): - string_tests.CommonTest.test_lower(self) + string_tests.StringLikeTest.test_lower(self) self.assertEqual('\U00010427'.lower(), '\U0001044F') self.assertEqual('\U00010427\U00010427'.lower(), '\U0001044F\U0001044F') @@ -897,7 +896,7 @@ def test_casefold(self): self.assertEqual('\u00b5'.casefold(), '\u03bc') def test_upper(self): - string_tests.CommonTest.test_upper(self) + string_tests.StringLikeTest.test_upper(self) self.assertEqual('\U0001044F'.upper(), '\U00010427') self.assertEqual('\U0001044F\U0001044F'.upper(), '\U00010427\U00010427') @@ -914,7 +913,7 @@ def test_upper(self): self.assertEqual('\u2177'.upper(), '\u2167') def test_capitalize(self): - string_tests.CommonTest.test_capitalize(self) + string_tests.StringLikeTest.test_capitalize(self) self.assertEqual('\U0001044F'.capitalize(), '\U00010427') self.assertEqual('\U0001044F\U0001044F'.capitalize(), '\U00010427\U0001044F') @@ -948,7 +947,7 @@ def test_title(self): self.assertEqual('A\u03a3A'.title(), 'A\u03c3a') def test_swapcase(self): - string_tests.CommonTest.test_swapcase(self) + string_tests.StringLikeTest.test_swapcase(self) self.assertEqual('\U0001044F'.swapcase(), '\U00010427') self.assertEqual('\U00010427'.swapcase(), '\U0001044F') self.assertEqual('\U0001044F\U0001044F'.swapcase(), @@ -974,7 +973,7 @@ def test_swapcase(self): self.assertEqual('\u1fd2'.swapcase(), '\u0399\u0308\u0300') def test_center(self): - string_tests.CommonTest.test_center(self) + string_tests.StringLikeTest.test_center(self) self.assertEqual('x'.center(2, '\U0010FFFF'), 'x\U0010FFFF') self.assertEqual('x'.center(3, '\U0010FFFF'), @@ -1475,7 +1474,7 @@ def __format__(self, spec): self.assertEqual('{f:{}}{}{g}'.format(2, 4, f=1, g='g'), ' 14g') def test_formatting(self): - string_tests.MixinStrUnicodeUserStringTest.test_formatting(self) + string_tests.StringLikeTest.test_formatting(self) # Testing Unicode formatting strings... self.assertEqual("%s, %s" % ("abc", "abc"), 'abc, abc') self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", 1, 2, 3), 'abc, abc, 1, 2.000000, 3.00') diff --git a/Lib/test/test_userstring.py b/Lib/test/test_userstring.py index 51b4f6041e49bf..74df52f5412af0 100644 --- a/Lib/test/test_userstring.py +++ b/Lib/test/test_userstring.py @@ -7,8 +7,7 @@ from collections import UserString class UserStringTest( - string_tests.CommonTest, - string_tests.MixinStrUnicodeUserStringTest, + string_tests.StringLikeTest, unittest.TestCase ): diff --git a/Misc/NEWS.d/next/Tests/2023-05-19-08-06-06.gh-issue-81005.-q7m9W.rst b/Misc/NEWS.d/next/Tests/2023-05-19-08-06-06.gh-issue-81005.-q7m9W.rst new file mode 100644 index 00000000000000..dfb653241e2607 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-05-19-08-06-06.gh-issue-81005.-q7m9W.rst @@ -0,0 +1,2 @@ +String tests are modified to reflect that ``str`` and ``unicode`` are merged +in Python 3. Patch by Daniel Fortunov. From ae00b810d1d3ad7f1f7e226b02ece37c986330e7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 23 May 2023 19:40:02 +0200 Subject: [PATCH 39/47] gh-104780: Remove 2to3 program and lib2to3 module (#104781) * Remove the Tools/scripts/2to3 script. * Remove the Lib/test/test_lib2to3/ directory. * Doc/tools/extensions/pyspecific.py: remove the "2to3fixer" object type. * Makefile and PC/layout/main.py no longer compile lib2to3 grammar files. * Update Makefile for 2to3 removal. --- .gitignore | 1 - Doc/glossary.rst | 9 - Doc/library/2to3.rst | 489 -- Doc/library/development.rst | 4 +- Doc/tools/.nitignore | 1 - Doc/tools/extensions/pyspecific.py | 1 - Doc/whatsnew/3.0.rst | 2 +- Doc/whatsnew/3.11.rst | 2 +- Doc/whatsnew/3.13.rst | 4 + Doc/whatsnew/3.9.rst | 4 +- Lib/lib2to3/Grammar.txt | 196 - Lib/lib2to3/PatternGrammar.txt | 28 - Lib/lib2to3/__init__.py | 8 - Lib/lib2to3/__main__.py | 4 - Lib/lib2to3/btm_matcher.py | 163 - Lib/lib2to3/btm_utils.py | 280 - Lib/lib2to3/fixer_base.py | 186 - Lib/lib2to3/fixer_util.py | 453 -- Lib/lib2to3/fixes/__init__.py | 1 - Lib/lib2to3/fixes/fix_apply.py | 68 - Lib/lib2to3/fixes/fix_asserts.py | 34 - Lib/lib2to3/fixes/fix_basestring.py | 14 - Lib/lib2to3/fixes/fix_buffer.py | 22 - Lib/lib2to3/fixes/fix_dict.py | 106 - Lib/lib2to3/fixes/fix_except.py | 93 - Lib/lib2to3/fixes/fix_exec.py | 39 - Lib/lib2to3/fixes/fix_execfile.py | 53 - Lib/lib2to3/fixes/fix_exitfunc.py | 72 - Lib/lib2to3/fixes/fix_filter.py | 94 - Lib/lib2to3/fixes/fix_funcattrs.py | 21 - Lib/lib2to3/fixes/fix_future.py | 22 - Lib/lib2to3/fixes/fix_getcwdu.py | 19 - Lib/lib2to3/fixes/fix_has_key.py | 109 - Lib/lib2to3/fixes/fix_idioms.py | 152 - Lib/lib2to3/fixes/fix_import.py | 99 - Lib/lib2to3/fixes/fix_imports.py | 145 - Lib/lib2to3/fixes/fix_imports2.py | 16 - Lib/lib2to3/fixes/fix_input.py | 26 - Lib/lib2to3/fixes/fix_intern.py | 39 - Lib/lib2to3/fixes/fix_isinstance.py | 52 - Lib/lib2to3/fixes/fix_itertools.py | 43 - Lib/lib2to3/fixes/fix_itertools_imports.py | 57 - Lib/lib2to3/fixes/fix_long.py | 19 - Lib/lib2to3/fixes/fix_map.py | 110 - Lib/lib2to3/fixes/fix_metaclass.py | 228 - Lib/lib2to3/fixes/fix_methodattrs.py | 24 - Lib/lib2to3/fixes/fix_ne.py | 23 - Lib/lib2to3/fixes/fix_next.py | 103 - Lib/lib2to3/fixes/fix_nonzero.py | 21 - Lib/lib2to3/fixes/fix_numliterals.py | 28 - Lib/lib2to3/fixes/fix_operator.py | 97 - Lib/lib2to3/fixes/fix_paren.py | 44 - Lib/lib2to3/fixes/fix_print.py | 87 - Lib/lib2to3/fixes/fix_raise.py | 90 - Lib/lib2to3/fixes/fix_raw_input.py | 17 - Lib/lib2to3/fixes/fix_reduce.py | 35 - Lib/lib2to3/fixes/fix_reload.py | 36 - Lib/lib2to3/fixes/fix_renames.py | 70 - Lib/lib2to3/fixes/fix_repr.py | 23 - Lib/lib2to3/fixes/fix_set_literal.py | 53 - Lib/lib2to3/fixes/fix_standarderror.py | 18 - Lib/lib2to3/fixes/fix_sys_exc.py | 30 - Lib/lib2to3/fixes/fix_throw.py | 56 - Lib/lib2to3/fixes/fix_tuple_params.py | 175 - Lib/lib2to3/fixes/fix_types.py | 61 - Lib/lib2to3/fixes/fix_unicode.py | 42 - Lib/lib2to3/fixes/fix_urllib.py | 196 - Lib/lib2to3/fixes/fix_ws_comma.py | 39 - Lib/lib2to3/fixes/fix_xrange.py | 73 - Lib/lib2to3/fixes/fix_xreadlines.py | 25 - Lib/lib2to3/fixes/fix_zip.py | 46 - Lib/lib2to3/main.py | 273 - Lib/lib2to3/patcomp.py | 204 - Lib/lib2to3/pgen2/__init__.py | 4 - Lib/lib2to3/pgen2/conv.py | 257 - Lib/lib2to3/pgen2/driver.py | 177 - Lib/lib2to3/pgen2/grammar.py | 189 - Lib/lib2to3/pgen2/literals.py | 60 - Lib/lib2to3/pgen2/parse.py | 204 - Lib/lib2to3/pgen2/pgen.py | 386 -- Lib/lib2to3/pgen2/token.py | 86 - Lib/lib2to3/pgen2/tokenize.py | 564 -- Lib/lib2to3/pygram.py | 43 - Lib/lib2to3/pytree.py | 853 --- Lib/lib2to3/refactor.py | 732 --- Lib/test/test_lib2to3/__init__.py | 11 - Lib/test/test_lib2to3/__main__.py | 4 - Lib/test/test_lib2to3/data/README | 6 - Lib/test/test_lib2to3/data/bom.py | 2 - Lib/test/test_lib2to3/data/crlf.py | 3 - .../test_lib2to3/data/different_encoding.py | 6 - Lib/test/test_lib2to3/data/false_encoding.py | 2 - .../test_lib2to3/data/fixers/bad_order.py | 5 - .../data/fixers/myfixes/__init__.py | 0 .../data/fixers/myfixes/fix_explicit.py | 6 - .../data/fixers/myfixes/fix_first.py | 6 - .../data/fixers/myfixes/fix_last.py | 7 - .../data/fixers/myfixes/fix_parrot.py | 13 - .../data/fixers/myfixes/fix_preorder.py | 6 - .../test_lib2to3/data/fixers/no_fixer_cls.py | 1 - .../data/fixers/parrot_example.py | 2 - .../test_lib2to3/data/infinite_recursion.py | 2669 ---------- .../test_lib2to3/data/py2_test_grammar.py | 971 ---- .../test_lib2to3/data/py3_test_grammar.py | 956 ---- Lib/test/test_lib2to3/pytree_idempotency.py | 94 - Lib/test/test_lib2to3/support.py | 69 - Lib/test/test_lib2to3/test_all_fixers.py | 41 - Lib/test/test_lib2to3/test_fixers.py | 4649 ----------------- Lib/test/test_lib2to3/test_main.py | 139 - Lib/test/test_lib2to3/test_parser.py | 718 --- Lib/test/test_lib2to3/test_pytree.py | 472 -- Lib/test/test_lib2to3/test_refactor.py | 337 -- Lib/test/test_lib2to3/test_util.py | 591 --- Lib/test/test_tools/test_sundry.py | 8 - Mac/BuildScript/scripts/postflight.framework | 4 +- Mac/Makefile.in | 2 - Makefile.pre.in | 25 +- Misc/NEWS.d/3.11.0a1.rst | 2 +- Misc/NEWS.d/3.8.0a1.rst | 4 +- Misc/NEWS.d/3.9.0a3.rst | 2 +- Misc/NEWS.d/3.9.0a6.rst | 2 +- ...-05-23-03-36-47.gh-issue-104780.P4e3Yf.rst | 2 + PC/layout/main.py | 23 - PC/layout/support/props.py | 2 - PC/layout/support/python.props | 2 - Python/stdlib_module_names.h | 1 - Tools/README | 6 +- .../msi/bundle/packagegroups/postinstall.wxs | 2 +- Tools/msi/lib/lib_files.wxs | 4 - Tools/msi/msi.props | 5 +- Tools/peg_generator/Makefile | 3 +- Tools/peg_generator/scripts/benchmark.py | 1 - Tools/scripts/2to3 | 5 - Tools/scripts/README | 1 - Tools/wasm/wasm_assets.py | 2 - 135 files changed, 26 insertions(+), 21000 deletions(-) delete mode 100644 Doc/library/2to3.rst delete mode 100644 Lib/lib2to3/Grammar.txt delete mode 100644 Lib/lib2to3/PatternGrammar.txt delete mode 100644 Lib/lib2to3/__init__.py delete mode 100644 Lib/lib2to3/__main__.py delete mode 100644 Lib/lib2to3/btm_matcher.py delete mode 100644 Lib/lib2to3/btm_utils.py delete mode 100644 Lib/lib2to3/fixer_base.py delete mode 100644 Lib/lib2to3/fixer_util.py delete mode 100644 Lib/lib2to3/fixes/__init__.py delete mode 100644 Lib/lib2to3/fixes/fix_apply.py delete mode 100644 Lib/lib2to3/fixes/fix_asserts.py delete mode 100644 Lib/lib2to3/fixes/fix_basestring.py delete mode 100644 Lib/lib2to3/fixes/fix_buffer.py delete mode 100644 Lib/lib2to3/fixes/fix_dict.py delete mode 100644 Lib/lib2to3/fixes/fix_except.py delete mode 100644 Lib/lib2to3/fixes/fix_exec.py delete mode 100644 Lib/lib2to3/fixes/fix_execfile.py delete mode 100644 Lib/lib2to3/fixes/fix_exitfunc.py delete mode 100644 Lib/lib2to3/fixes/fix_filter.py delete mode 100644 Lib/lib2to3/fixes/fix_funcattrs.py delete mode 100644 Lib/lib2to3/fixes/fix_future.py delete mode 100644 Lib/lib2to3/fixes/fix_getcwdu.py delete mode 100644 Lib/lib2to3/fixes/fix_has_key.py delete mode 100644 Lib/lib2to3/fixes/fix_idioms.py delete mode 100644 Lib/lib2to3/fixes/fix_import.py delete mode 100644 Lib/lib2to3/fixes/fix_imports.py delete mode 100644 Lib/lib2to3/fixes/fix_imports2.py delete mode 100644 Lib/lib2to3/fixes/fix_input.py delete mode 100644 Lib/lib2to3/fixes/fix_intern.py delete mode 100644 Lib/lib2to3/fixes/fix_isinstance.py delete mode 100644 Lib/lib2to3/fixes/fix_itertools.py delete mode 100644 Lib/lib2to3/fixes/fix_itertools_imports.py delete mode 100644 Lib/lib2to3/fixes/fix_long.py delete mode 100644 Lib/lib2to3/fixes/fix_map.py delete mode 100644 Lib/lib2to3/fixes/fix_metaclass.py delete mode 100644 Lib/lib2to3/fixes/fix_methodattrs.py delete mode 100644 Lib/lib2to3/fixes/fix_ne.py delete mode 100644 Lib/lib2to3/fixes/fix_next.py delete mode 100644 Lib/lib2to3/fixes/fix_nonzero.py delete mode 100644 Lib/lib2to3/fixes/fix_numliterals.py delete mode 100644 Lib/lib2to3/fixes/fix_operator.py delete mode 100644 Lib/lib2to3/fixes/fix_paren.py delete mode 100644 Lib/lib2to3/fixes/fix_print.py delete mode 100644 Lib/lib2to3/fixes/fix_raise.py delete mode 100644 Lib/lib2to3/fixes/fix_raw_input.py delete mode 100644 Lib/lib2to3/fixes/fix_reduce.py delete mode 100644 Lib/lib2to3/fixes/fix_reload.py delete mode 100644 Lib/lib2to3/fixes/fix_renames.py delete mode 100644 Lib/lib2to3/fixes/fix_repr.py delete mode 100644 Lib/lib2to3/fixes/fix_set_literal.py delete mode 100644 Lib/lib2to3/fixes/fix_standarderror.py delete mode 100644 Lib/lib2to3/fixes/fix_sys_exc.py delete mode 100644 Lib/lib2to3/fixes/fix_throw.py delete mode 100644 Lib/lib2to3/fixes/fix_tuple_params.py delete mode 100644 Lib/lib2to3/fixes/fix_types.py delete mode 100644 Lib/lib2to3/fixes/fix_unicode.py delete mode 100644 Lib/lib2to3/fixes/fix_urllib.py delete mode 100644 Lib/lib2to3/fixes/fix_ws_comma.py delete mode 100644 Lib/lib2to3/fixes/fix_xrange.py delete mode 100644 Lib/lib2to3/fixes/fix_xreadlines.py delete mode 100644 Lib/lib2to3/fixes/fix_zip.py delete mode 100644 Lib/lib2to3/main.py delete mode 100644 Lib/lib2to3/patcomp.py delete mode 100644 Lib/lib2to3/pgen2/__init__.py delete mode 100644 Lib/lib2to3/pgen2/conv.py delete mode 100644 Lib/lib2to3/pgen2/driver.py delete mode 100644 Lib/lib2to3/pgen2/grammar.py delete mode 100644 Lib/lib2to3/pgen2/literals.py delete mode 100644 Lib/lib2to3/pgen2/parse.py delete mode 100644 Lib/lib2to3/pgen2/pgen.py delete mode 100755 Lib/lib2to3/pgen2/token.py delete mode 100644 Lib/lib2to3/pgen2/tokenize.py delete mode 100644 Lib/lib2to3/pygram.py delete mode 100644 Lib/lib2to3/pytree.py delete mode 100644 Lib/lib2to3/refactor.py delete mode 100644 Lib/test/test_lib2to3/__init__.py delete mode 100644 Lib/test/test_lib2to3/__main__.py delete mode 100644 Lib/test/test_lib2to3/data/README delete mode 100644 Lib/test/test_lib2to3/data/bom.py delete mode 100644 Lib/test/test_lib2to3/data/crlf.py delete mode 100755 Lib/test/test_lib2to3/data/different_encoding.py delete mode 100755 Lib/test/test_lib2to3/data/false_encoding.py delete mode 100644 Lib/test/test_lib2to3/data/fixers/bad_order.py delete mode 100644 Lib/test/test_lib2to3/data/fixers/myfixes/__init__.py delete mode 100644 Lib/test/test_lib2to3/data/fixers/myfixes/fix_explicit.py delete mode 100644 Lib/test/test_lib2to3/data/fixers/myfixes/fix_first.py delete mode 100644 Lib/test/test_lib2to3/data/fixers/myfixes/fix_last.py delete mode 100644 Lib/test/test_lib2to3/data/fixers/myfixes/fix_parrot.py delete mode 100644 Lib/test/test_lib2to3/data/fixers/myfixes/fix_preorder.py delete mode 100644 Lib/test/test_lib2to3/data/fixers/no_fixer_cls.py delete mode 100644 Lib/test/test_lib2to3/data/fixers/parrot_example.py delete mode 100644 Lib/test/test_lib2to3/data/infinite_recursion.py delete mode 100644 Lib/test/test_lib2to3/data/py2_test_grammar.py delete mode 100644 Lib/test/test_lib2to3/data/py3_test_grammar.py delete mode 100755 Lib/test/test_lib2to3/pytree_idempotency.py delete mode 100644 Lib/test/test_lib2to3/support.py delete mode 100644 Lib/test/test_lib2to3/test_all_fixers.py delete mode 100644 Lib/test/test_lib2to3/test_fixers.py delete mode 100644 Lib/test/test_lib2to3/test_main.py delete mode 100644 Lib/test/test_lib2to3/test_parser.py delete mode 100644 Lib/test/test_lib2to3/test_pytree.py delete mode 100644 Lib/test/test_lib2to3/test_refactor.py delete mode 100644 Lib/test/test_lib2to3/test_util.py create mode 100644 Misc/NEWS.d/next/Library/2023-05-23-03-36-47.gh-issue-104780.P4e3Yf.rst delete mode 100755 Tools/scripts/2to3 diff --git a/.gitignore b/.gitignore index ef7642b09bc5d2..dddf28da016192 100644 --- a/.gitignore +++ b/.gitignore @@ -61,7 +61,6 @@ Doc/.venv/ Doc/env/ Doc/.env/ Include/pydtrace_probes.h -Lib/lib2to3/*.pickle Lib/site-packages/* !Lib/site-packages/README.txt Lib/test/data/* diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 53e8cdcae1cd66..8c493f823a6fae 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -22,15 +22,6 @@ Glossary * The :const:`Ellipsis` built-in constant. - 2to3 - A tool that tries to convert Python 2.x code to Python 3.x code by - handling most of the incompatibilities which can be detected by parsing the - source and traversing the parse tree. - - 2to3 is available in the standard library as :mod:`lib2to3`; a standalone - entry point is provided as :file:`Tools/scripts/2to3`. See - :ref:`2to3-reference`. - abstract base class Abstract base classes complement :term:`duck-typing` by providing a way to define interfaces when other techniques like diff --git a/Doc/library/2to3.rst b/Doc/library/2to3.rst deleted file mode 100644 index d85ad94e9b7fe4..00000000000000 --- a/Doc/library/2to3.rst +++ /dev/null @@ -1,489 +0,0 @@ -.. _2to3-reference: - -2to3 --- Automated Python 2 to 3 code translation -================================================= - -.. sectionauthor:: Benjamin Peterson - -2to3 is a Python program that reads Python 2.x source code and applies a series -of *fixers* to transform it into valid Python 3.x code. The standard library -contains a rich set of fixers that will handle almost all code. 2to3 supporting -library :mod:`lib2to3` is, however, a flexible and generic library, so it is -possible to write your own fixers for 2to3. - -.. deprecated-removed:: 3.11 3.13 - The ``lib2to3`` module was marked pending for deprecation in Python 3.9 - (raising :exc:`PendingDeprecationWarning` on import) and fully deprecated - in Python 3.11 (raising :exc:`DeprecationWarning`). The ``2to3`` tool is - part of that. It will be removed in Python 3.13. - -.. _2to3-using: - -Using 2to3 ----------- - -2to3 will usually be installed with the Python interpreter as a script. It is -also located in the :file:`Tools/scripts` directory of the Python root. - -2to3's basic arguments are a list of files or directories to transform. The -directories are recursively traversed for Python sources. - -Here is a sample Python 2.x source file, :file:`example.py`:: - - def greet(name): - print "Hello, {0}!".format(name) - print "What's your name?" - name = raw_input() - greet(name) - -It can be converted to Python 3.x code via 2to3 on the command line: - -.. code-block:: shell-session - - $ 2to3 example.py - -A diff against the original source file is printed. 2to3 can also write the -needed modifications right back to the source file. (A backup of the original -file is made unless :option:`!-n` is also given.) Writing the changes back is -enabled with the :option:`!-w` flag: - -.. code-block:: shell-session - - $ 2to3 -w example.py - -After transformation, :file:`example.py` looks like this:: - - def greet(name): - print("Hello, {0}!".format(name)) - print("What's your name?") - name = input() - greet(name) - -Comments and exact indentation are preserved throughout the translation process. - -By default, 2to3 runs a set of :ref:`predefined fixers <2to3-fixers>`. The -:option:`!-l` flag lists all available fixers. An explicit set of fixers to run -can be given with :option:`!-f`. Likewise the :option:`!-x` explicitly disables a -fixer. The following example runs only the ``imports`` and ``has_key`` fixers: - -.. code-block:: shell-session - - $ 2to3 -f imports -f has_key example.py - -This command runs every fixer except the ``apply`` fixer: - -.. code-block:: shell-session - - $ 2to3 -x apply example.py - -Some fixers are *explicit*, meaning they aren't run by default and must be -listed on the command line to be run. Here, in addition to the default fixers, -the ``idioms`` fixer is run: - -.. code-block:: shell-session - - $ 2to3 -f all -f idioms example.py - -Notice how passing ``all`` enables all default fixers. - -Sometimes 2to3 will find a place in your source code that needs to be changed, -but 2to3 cannot fix automatically. In this case, 2to3 will print a warning -beneath the diff for a file. You should address the warning in order to have -compliant 3.x code. - -2to3 can also refactor doctests. To enable this mode, use the :option:`!-d` -flag. Note that *only* doctests will be refactored. This also doesn't require -the module to be valid Python. For example, doctest like examples in a reST -document could also be refactored with this option. - -The :option:`!-v` option enables output of more information on the translation -process. - -Since some print statements can be parsed as function calls or statements, 2to3 -cannot always read files containing the print function. When 2to3 detects the -presence of the ``from __future__ import print_function`` compiler directive, it -modifies its internal grammar to interpret :func:`print` as a function. This -change can also be enabled manually with the :option:`!-p` flag. Use -:option:`!-p` to run fixers on code that already has had its print statements -converted. Also :option:`!-e` can be used to make :func:`exec` a function. - -The :option:`!-o` or :option:`!--output-dir` option allows specification of an -alternate directory for processed output files to be written to. The -:option:`!-n` flag is required when using this as backup files do not make sense -when not overwriting the input files. - -.. versionadded:: 3.2.3 - The :option:`!-o` option was added. - -The :option:`!-W` or :option:`!--write-unchanged-files` flag tells 2to3 to always -write output files even if no changes were required to the file. This is most -useful with :option:`!-o` so that an entire Python source tree is copied with -translation from one directory to another. -This option implies the :option:`!-w` flag as it would not make sense otherwise. - -.. versionadded:: 3.2.3 - The :option:`!-W` flag was added. - -The :option:`!--add-suffix` option specifies a string to append to all output -filenames. The :option:`!-n` flag is required when specifying this as backups -are not necessary when writing to different filenames. Example: - -.. code-block:: shell-session - - $ 2to3 -n -W --add-suffix=3 example.py - -Will cause a converted file named ``example.py3`` to be written. - -.. versionadded:: 3.2.3 - The :option:`!--add-suffix` option was added. - -To translate an entire project from one directory tree to another use: - -.. code-block:: shell-session - - $ 2to3 --output-dir=python3-version/mycode -W -n python2-version/mycode - - -.. _2to3-fixers: - -Fixers ------- - -Each step of transforming code is encapsulated in a fixer. The command ``2to3 --l`` lists them. As :ref:`documented above <2to3-using>`, each can be turned on -and off individually. They are described here in more detail. - - -.. 2to3fixer:: apply - - Removes usage of :func:`apply`. For example ``apply(function, *args, - **kwargs)`` is converted to ``function(*args, **kwargs)``. - -.. 2to3fixer:: asserts - - Replaces deprecated :mod:`unittest` method names with the correct ones. - - ================================ ========================================== - From To - ================================ ========================================== - ``failUnlessEqual(a, b)`` :meth:`assertEqual(a, b) - ` - ``assertEquals(a, b)`` :meth:`assertEqual(a, b) - ` - ``failIfEqual(a, b)`` :meth:`assertNotEqual(a, b) - ` - ``assertNotEquals(a, b)`` :meth:`assertNotEqual(a, b) - ` - ``failUnless(a)`` :meth:`assertTrue(a) - ` - ``assert_(a)`` :meth:`assertTrue(a) - ` - ``failIf(a)`` :meth:`assertFalse(a) - ` - ``failUnlessRaises(exc, cal)`` :meth:`assertRaises(exc, cal) - ` - ``failUnlessAlmostEqual(a, b)`` :meth:`assertAlmostEqual(a, b) - ` - ``assertAlmostEquals(a, b)`` :meth:`assertAlmostEqual(a, b) - ` - ``failIfAlmostEqual(a, b)`` :meth:`assertNotAlmostEqual(a, b) - ` - ``assertNotAlmostEquals(a, b)`` :meth:`assertNotAlmostEqual(a, b) - ` - ================================ ========================================== - -.. 2to3fixer:: basestring - - Converts :class:`basestring` to :class:`str`. - -.. 2to3fixer:: buffer - - Converts :class:`buffer` to :class:`memoryview`. This fixer is optional - because the :class:`memoryview` API is similar but not exactly the same as - that of :class:`buffer`. - -.. 2to3fixer:: dict - - Fixes dictionary iteration methods. :meth:`dict.iteritems` is converted to - :meth:`dict.items`, :meth:`dict.iterkeys` to :meth:`dict.keys`, and - :meth:`dict.itervalues` to :meth:`dict.values`. Similarly, - :meth:`dict.viewitems`, :meth:`dict.viewkeys` and :meth:`dict.viewvalues` are - converted respectively to :meth:`dict.items`, :meth:`dict.keys` and - :meth:`dict.values`. It also wraps existing usages of :meth:`dict.items`, - :meth:`dict.keys`, and :meth:`dict.values` in a call to :class:`list`. - -.. 2to3fixer:: except - - Converts ``except X, T`` to ``except X as T``. - -.. 2to3fixer:: exec - - Converts the ``exec`` statement to the :func:`exec` function. - -.. 2to3fixer:: execfile - - Removes usage of :func:`execfile`. The argument to :func:`execfile` is - wrapped in calls to :func:`open`, :func:`compile`, and :func:`exec`. - -.. 2to3fixer:: exitfunc - - Changes assignment of :attr:`sys.exitfunc` to use of the :mod:`atexit` - module. - -.. 2to3fixer:: filter - - Wraps :func:`filter` usage in a :class:`list` call. - -.. 2to3fixer:: funcattrs - - Fixes function attributes that have been renamed. For example, - ``my_function.func_closure`` is converted to ``my_function.__closure__``. - -.. 2to3fixer:: future - - Removes ``from __future__ import new_feature`` statements. - -.. 2to3fixer:: getcwdu - - Renames :func:`os.getcwdu` to :func:`os.getcwd`. - -.. 2to3fixer:: has_key - - Changes ``dict.has_key(key)`` to ``key in dict``. - -.. 2to3fixer:: idioms - - This optional fixer performs several transformations that make Python code - more idiomatic. Type comparisons like ``type(x) is SomeClass`` and - ``type(x) == SomeClass`` are converted to ``isinstance(x, SomeClass)``. - ``while 1`` becomes ``while True``. This fixer also tries to make use of - :func:`sorted` in appropriate places. For example, this block :: - - L = list(some_iterable) - L.sort() - - is changed to :: - - L = sorted(some_iterable) - -.. 2to3fixer:: import - - Detects sibling imports and converts them to relative imports. - -.. 2to3fixer:: imports - - Handles module renames in the standard library. - -.. 2to3fixer:: imports2 - - Handles other modules renames in the standard library. It is separate from - the :2to3fixer:`imports` fixer only because of technical limitations. - -.. 2to3fixer:: input - - Converts ``input(prompt)`` to ``eval(input(prompt))``. - -.. 2to3fixer:: intern - - Converts :func:`intern` to :func:`sys.intern`. - -.. 2to3fixer:: isinstance - - Fixes duplicate types in the second argument of :func:`isinstance`. For - example, ``isinstance(x, (int, int))`` is converted to ``isinstance(x, - int)`` and ``isinstance(x, (int, float, int))`` is converted to - ``isinstance(x, (int, float))``. - -.. 2to3fixer:: itertools_imports - - Removes imports of :func:`itertools.ifilter`, :func:`itertools.izip`, and - :func:`itertools.imap`. Imports of :func:`itertools.ifilterfalse` are also - changed to :func:`itertools.filterfalse`. - -.. 2to3fixer:: itertools - - Changes usage of :func:`itertools.ifilter`, :func:`itertools.izip`, and - :func:`itertools.imap` to their built-in equivalents. - :func:`itertools.ifilterfalse` is changed to :func:`itertools.filterfalse`. - -.. 2to3fixer:: long - - Renames :class:`long` to :class:`int`. - -.. 2to3fixer:: map - - Wraps :func:`map` in a :class:`list` call. It also changes ``map(None, x)`` - to ``list(x)``. Using ``from future_builtins import map`` disables this - fixer. - -.. 2to3fixer:: metaclass - - Converts the old metaclass syntax (``__metaclass__ = Meta`` in the class - body) to the new (``class X(metaclass=Meta)``). - -.. 2to3fixer:: methodattrs - - Fixes old method attribute names. For example, ``meth.im_func`` is converted - to ``meth.__func__``. - -.. 2to3fixer:: ne - - Converts the old not-equal syntax, ``<>``, to ``!=``. - -.. 2to3fixer:: next - - Converts the use of iterator's :meth:`~iterator.next` methods to the - :func:`next` function. It also renames :meth:`next` methods to - :meth:`~iterator.__next__`. - -.. 2to3fixer:: nonzero - - Renames definitions of methods called :meth:`__nonzero__` - to :meth:`~object.__bool__`. - -.. 2to3fixer:: numliterals - - Converts octal literals into the new syntax. - -.. 2to3fixer:: operator - - Converts calls to various functions in the :mod:`operator` module to other, - but equivalent, function calls. When needed, the appropriate ``import`` - statements are added, e.g. ``import collections.abc``. The following mapping - are made: - - ================================== ============================================= - From To - ================================== ============================================= - ``operator.isCallable(obj)`` ``callable(obj)`` - ``operator.sequenceIncludes(obj)`` ``operator.contains(obj)`` - ``operator.isSequenceType(obj)`` ``isinstance(obj, collections.abc.Sequence)`` - ``operator.isMappingType(obj)`` ``isinstance(obj, collections.abc.Mapping)`` - ``operator.isNumberType(obj)`` ``isinstance(obj, numbers.Number)`` - ``operator.repeat(obj, n)`` ``operator.mul(obj, n)`` - ``operator.irepeat(obj, n)`` ``operator.imul(obj, n)`` - ================================== ============================================= - -.. 2to3fixer:: paren - - Add extra parenthesis where they are required in list comprehensions. For - example, ``[x for x in 1, 2]`` becomes ``[x for x in (1, 2)]``. - -.. 2to3fixer:: print - - Converts the ``print`` statement to the :func:`print` function. - -.. 2to3fixer:: raise - - Converts ``raise E, V`` to ``raise E(V)``, and ``raise E, V, T`` to ``raise - E(V).with_traceback(T)``. If ``E`` is a tuple, the translation will be - incorrect because substituting tuples for exceptions has been removed in 3.0. - -.. 2to3fixer:: raw_input - - Converts :func:`raw_input` to :func:`input`. - -.. 2to3fixer:: reduce - - Handles the move of :func:`reduce` to :func:`functools.reduce`. - -.. 2to3fixer:: reload - - Converts :func:`reload` to :func:`importlib.reload`. - -.. 2to3fixer:: renames - - Changes :data:`sys.maxint` to :data:`sys.maxsize`. - -.. 2to3fixer:: repr - - Replaces backtick repr with the :func:`repr` function. - -.. 2to3fixer:: set_literal - - Replaces use of the :class:`set` constructor with set literals. This fixer - is optional. - -.. 2to3fixer:: standarderror - - Renames :exc:`StandardError` to :exc:`Exception`. - -.. 2to3fixer:: sys_exc - - Changes the deprecated :data:`sys.exc_value`, :data:`sys.exc_type`, - :data:`sys.exc_traceback` to use :func:`sys.exc_info`. - -.. 2to3fixer:: throw - - Fixes the API change in generator's :meth:`throw` method. - -.. 2to3fixer:: tuple_params - - Removes implicit tuple parameter unpacking. This fixer inserts temporary - variables. - -.. 2to3fixer:: types - - Fixes code broken from the removal of some members in the :mod:`types` - module. - -.. 2to3fixer:: unicode - - Renames :class:`unicode` to :class:`str`. - -.. 2to3fixer:: urllib - - Handles the rename of :mod:`urllib` and :mod:`urllib2` to the :mod:`urllib` - package. - -.. 2to3fixer:: ws_comma - - Removes excess whitespace from comma separated items. This fixer is - optional. - -.. 2to3fixer:: xrange - - Renames :func:`xrange` to :func:`range` and wraps existing :func:`range` - calls with :class:`list`. - -.. 2to3fixer:: xreadlines - - Changes ``for x in file.xreadlines()`` to ``for x in file``. - -.. 2to3fixer:: zip - - Wraps :func:`zip` usage in a :class:`list` call. This is disabled when - ``from future_builtins import zip`` appears. - - -:mod:`lib2to3` --- 2to3's library ---------------------------------- - -.. module:: lib2to3 - :synopsis: The 2to3 library - -.. moduleauthor:: Guido van Rossum -.. moduleauthor:: Collin Winter -.. moduleauthor:: Benjamin Peterson - -**Source code:** :source:`Lib/lib2to3/` - --------------- - -.. deprecated-removed:: 3.11 3.13 - Python 3.9 switched to a PEG parser (see :pep:`617`) while lib2to3 is - using a less flexible LL(1) parser. Python 3.10 includes new language - syntax that is not parsable by lib2to3's LL(1) parser (see :pep:`634`). - The ``lib2to3`` module was marked pending for deprecation in Python 3.9 - (raising :exc:`PendingDeprecationWarning` on import) and fully deprecated - in Python 3.11 (raising :exc:`DeprecationWarning`). - It will be removed from the standard library in Python 3.13. - Consider third-party alternatives such as `LibCST`_ or `parso`_. - -.. note:: - - The :mod:`lib2to3` API should be considered unstable and may change - drastically in the future. - -.. _LibCST: https://libcst.readthedocs.io/ -.. _parso: https://parso.readthedocs.io/ diff --git a/Doc/library/development.rst b/Doc/library/development.rst index 9edce758688e2d..b1979b921e7d5f 100644 --- a/Doc/library/development.rst +++ b/Doc/library/development.rst @@ -8,8 +8,7 @@ The modules described in this chapter help you write software. For example, the :mod:`pydoc` module takes a module and generates documentation based on the module's contents. The :mod:`doctest` and :mod:`unittest` modules contains frameworks for writing unit tests that automatically exercise code and verify -that the expected output is produced. :program:`2to3` can translate Python 2.x -source code into valid Python 3.x code. +that the expected output is produced. The list of modules described in this chapter is: @@ -23,5 +22,4 @@ The list of modules described in this chapter is: unittest.rst unittest.mock.rst unittest.mock-examples.rst - 2to3.rst test.rst diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 554e31ff51dd3e..7a711031d74e9a 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -72,7 +72,6 @@ Doc/howto/sorting.rst Doc/howto/unicode.rst Doc/howto/urllib2.rst Doc/install/index.rst -Doc/library/2to3.rst Doc/library/__future__.rst Doc/library/_thread.rst Doc/library/abc.rst diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 3a5b26f7779618..8a2eb07a69a69a 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -710,7 +710,6 @@ def setup(app): app.add_builder(PydocTopicsBuilder) app.add_object_type('opcode', 'opcode', '%s (opcode)', parse_opcode_signature) app.add_object_type('pdbcommand', 'pdbcmd', '%s (pdb command)', parse_pdb_command) - app.add_object_type('2to3fixer', '2to3fixer', '%s (2to3 fixer)') app.add_directive_to_domain('py', 'decorator', PyDecoratorFunction) app.add_directive_to_domain('py', 'decoratormethod', PyDecoratorMethod) app.add_directive_to_domain('py', 'coroutinefunction', PyCoroutineFunction) diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst index f9ac13036cbc8d..b8cd7c48b359b2 100644 --- a/Doc/whatsnew/3.0.rst +++ b/Doc/whatsnew/3.0.rst @@ -911,7 +911,7 @@ best strategy is the following: tests still pass. 3. Run the ``2to3`` source-to-source translator over your source code - tree. (See :ref:`2to3-reference` for more on this tool.) Run the + tree. Run the result of the translation under Python 3.0. Manually fix up any remaining issues, fixing problems until all tests pass again. diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 1a4a9936aca685..8aadc2a0a581f2 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -1748,7 +1748,7 @@ Modules warnings have now been updated to note they will be removed in Python 3.12. (Contributed by Hugo van Kemenade in :issue:`47022`.) -* The :mod:`lib2to3` package and :ref:`2to3 <2to3-reference>` tool +* The :mod:`!lib2to3` package and ``2to3`` tool are now deprecated and may not be able to parse Python 3.10 or newer. See :pep:`617`, introducing the new PEG parser, for details. (Contributed by Victor Stinner in :issue:`40360`.) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 602a865b5930e8..e0c3c2a3592ec7 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -107,6 +107,10 @@ Removed `Exscript `_ instead. (Contributed by Victor Stinner in :gh:`104773`.) +* Remove the ``2to3`` program and the :mod:`!lib2to3` module, + deprecated in Python 3.11. + (Contributed by Victor Stinner in :gh:`104780`.) + * Namespaces ``typing.io`` and ``typing.re``, deprecated in Python 3.8, are now removed. The items in those namespaces can be imported directly from :mod:`typing`. (Contributed by Sebastian Rittau in :gh:`92871`.) diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index fd86db96302356..532cabdd7a75be 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -935,10 +935,10 @@ Deprecated * :func:`smtpd.MailmanProxy` is now deprecated as it is unusable without an external module, ``mailman``. (Contributed by Samuel Colvin in :issue:`35800`.) -* The :mod:`lib2to3` module now emits a :exc:`PendingDeprecationWarning`. +* The :mod:`!lib2to3` module now emits a :exc:`PendingDeprecationWarning`. Python 3.9 switched to a PEG parser (see :pep:`617`), and Python 3.10 may include new language syntax that is not parsable by lib2to3's LL(1) parser. - The ``lib2to3`` module may be removed from the standard library in a future + The :mod:`!lib2to3` module may be removed from the standard library in a future Python version. Consider third-party alternatives such as `LibCST`_ or `parso`_. (Contributed by Carl Meyer in :issue:`40360`.) diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt deleted file mode 100644 index fa7b15061d941c..00000000000000 --- a/Lib/lib2to3/Grammar.txt +++ /dev/null @@ -1,196 +0,0 @@ -# Grammar for 2to3. This grammar supports Python 2.x and 3.x. - -# NOTE WELL: You should also follow all the steps listed at -# https://devguide.python.org/grammar/ - -# Start symbols for the grammar: -# file_input is a module or sequence of commands read from an input file; -# single_input is a single interactive statement; -# eval_input is the input for the eval() and input() functions. -# NB: compound_stmt in single_input is followed by extra NEWLINE! -file_input: (NEWLINE | stmt)* ENDMARKER -single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE -eval_input: testlist NEWLINE* ENDMARKER - -decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE -decorators: decorator+ -decorated: decorators (classdef | funcdef | async_funcdef) -async_funcdef: ASYNC funcdef -funcdef: 'def' NAME parameters ['->' test] ':' suite -parameters: '(' [typedargslist] ')' - -# The following definition for typedarglist is equivalent to this set of rules: -# -# arguments = argument (',' argument)* -# argument = tfpdef ['=' test] -# kwargs = '**' tname [','] -# args = '*' [tname] -# kwonly_kwargs = (',' argument)* [',' [kwargs]] -# args_kwonly_kwargs = args kwonly_kwargs | kwargs -# poskeyword_args_kwonly_kwargs = arguments [',' [args_kwonly_kwargs]] -# typedargslist_no_posonly = poskeyword_args_kwonly_kwargs | args_kwonly_kwargs -# typedarglist = arguments ',' '/' [',' [typedargslist_no_posonly]])|(typedargslist_no_posonly)" -# -# It needs to be fully expanded to allow our LL(1) parser to work on it. - -typedargslist: tfpdef ['=' test] (',' tfpdef ['=' test])* ',' '/' [ - ',' [((tfpdef ['=' test] ',')* ('*' [tname] (',' tname ['=' test])* - [',' ['**' tname [',']]] | '**' tname [',']) - | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])] - ] | ((tfpdef ['=' test] ',')* ('*' [tname] (',' tname ['=' test])* - [',' ['**' tname [',']]] | '**' tname [',']) - | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) - -tname: NAME [':' test] -tfpdef: tname | '(' tfplist ')' -tfplist: tfpdef (',' tfpdef)* [','] - -# The following definition for varargslist is equivalent to this set of rules: -# -# arguments = argument (',' argument )* -# argument = vfpdef ['=' test] -# kwargs = '**' vname [','] -# args = '*' [vname] -# kwonly_kwargs = (',' argument )* [',' [kwargs]] -# args_kwonly_kwargs = args kwonly_kwargs | kwargs -# poskeyword_args_kwonly_kwargs = arguments [',' [args_kwonly_kwargs]] -# vararglist_no_posonly = poskeyword_args_kwonly_kwargs | args_kwonly_kwargs -# varargslist = arguments ',' '/' [','[(vararglist_no_posonly)]] | (vararglist_no_posonly) -# -# It needs to be fully expanded to allow our LL(1) parser to work on it. - -varargslist: vfpdef ['=' test ](',' vfpdef ['=' test])* ',' '/' [',' [ - ((vfpdef ['=' test] ',')* ('*' [vname] (',' vname ['=' test])* - [',' ['**' vname [',']]] | '**' vname [',']) - | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) - ]] | ((vfpdef ['=' test] ',')* - ('*' [vname] (',' vname ['=' test])* [',' ['**' vname [',']]]| '**' vname [',']) - | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) - -vname: NAME -vfpdef: vname | '(' vfplist ')' -vfplist: vfpdef (',' vfpdef)* [','] - -stmt: simple_stmt | compound_stmt -simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE -small_stmt: (expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt | - import_stmt | global_stmt | exec_stmt | assert_stmt) -expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) | - ('=' (yield_expr|testlist_star_expr))*) -annassign: ':' test ['=' test] -testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [','] -augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' | - '<<=' | '>>=' | '**=' | '//=') -# For normal and annotated assignments, additional restrictions enforced by the interpreter -print_stmt: 'print' ( [ test (',' test)* [','] ] | - '>>' test [ (',' test)+ [','] ] ) -del_stmt: 'del' exprlist -pass_stmt: 'pass' -flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt -break_stmt: 'break' -continue_stmt: 'continue' -return_stmt: 'return' [testlist_star_expr] -yield_stmt: yield_expr -raise_stmt: 'raise' [test ['from' test | ',' test [',' test]]] -import_stmt: import_name | import_from -import_name: 'import' dotted_as_names -import_from: ('from' ('.'* dotted_name | '.'+) - 'import' ('*' | '(' import_as_names ')' | import_as_names)) -import_as_name: NAME ['as' NAME] -dotted_as_name: dotted_name ['as' NAME] -import_as_names: import_as_name (',' import_as_name)* [','] -dotted_as_names: dotted_as_name (',' dotted_as_name)* -dotted_name: NAME ('.' NAME)* -global_stmt: ('global' | 'nonlocal') NAME (',' NAME)* -exec_stmt: 'exec' expr ['in' test [',' test]] -assert_stmt: 'assert' test [',' test] - -compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt -async_stmt: ASYNC (funcdef | with_stmt | for_stmt) -if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite] -while_stmt: 'while' namedexpr_test ':' suite ['else' ':' suite] -for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] -try_stmt: ('try' ':' suite - ((except_clause ':' suite)+ - ['else' ':' suite] - ['finally' ':' suite] | - 'finally' ':' suite)) -with_stmt: 'with' with_item (',' with_item)* ':' suite -with_item: test ['as' expr] -with_var: 'as' expr -# NB compile.c makes sure that the default except clause is last -except_clause: 'except' [test [(',' | 'as') test]] -suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT - -# Backward compatibility cruft to support: -# [ x for x in lambda: True, lambda: False if x() ] -# even while also allowing: -# lambda x: 5 if x else 2 -# (But not a mix of the two) -testlist_safe: old_test [(',' old_test)+ [',']] -old_test: or_test | old_lambdef -old_lambdef: 'lambda' [varargslist] ':' old_test - -namedexpr_test: test [':=' test] -test: or_test ['if' or_test 'else' test] | lambdef -or_test: and_test ('or' and_test)* -and_test: not_test ('and' not_test)* -not_test: 'not' not_test | comparison -comparison: expr (comp_op expr)* -comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' -star_expr: '*' expr -expr: xor_expr ('|' xor_expr)* -xor_expr: and_expr ('^' and_expr)* -and_expr: shift_expr ('&' shift_expr)* -shift_expr: arith_expr (('<<'|'>>') arith_expr)* -arith_expr: term (('+'|'-') term)* -term: factor (('*'|'@'|'/'|'%'|'//') factor)* -factor: ('+'|'-'|'~') factor | power -power: [AWAIT] atom trailer* ['**' factor] -atom: ('(' [yield_expr|testlist_gexp] ')' | - '[' [listmaker] ']' | - '{' [dictsetmaker] '}' | - '`' testlist1 '`' | - NAME | NUMBER | STRING+ | '.' '.' '.') -listmaker: (namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] ) -testlist_gexp: (namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] ) -lambdef: 'lambda' [varargslist] ':' test -trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME -subscriptlist: subscript (',' subscript)* [','] -subscript: test | [test] ':' [test] [sliceop] -sliceop: ':' [test] -exprlist: (expr|star_expr) (',' (expr|star_expr))* [','] -testlist: test (',' test)* [','] -dictsetmaker: ( ((test ':' test | '**' expr) - (comp_for | (',' (test ':' test | '**' expr))* [','])) | - ((test | star_expr) - (comp_for | (',' (test | star_expr))* [','])) ) - -classdef: 'class' NAME ['(' [arglist] ')'] ':' suite - -arglist: argument (',' argument)* [','] - -# "test '=' test" is really "keyword '=' test", but we have no such token. -# These need to be in a single rule to avoid grammar that is ambiguous -# to our LL(1) parser. Even though 'test' includes '*expr' in star_expr, -# we explicitly match '*' here, too, to give it proper precedence. -# Illegal combinations and orderings are blocked in ast.c: -# multiple (test comp_for) arguments are blocked; keyword unpackings -# that precede iterable unpackings are blocked; etc. -argument: ( test [comp_for] | - test ':=' test | - test '=' test | - '**' test | - '*' test ) - -comp_iter: comp_for | comp_if -comp_for: [ASYNC] 'for' exprlist 'in' testlist_safe [comp_iter] -comp_if: 'if' old_test [comp_iter] - -testlist1: test (',' test)* - -# not used in grammar, but may appear in "node" passed from Parser to Compiler -encoding_decl: NAME - -yield_expr: 'yield' [yield_arg] -yield_arg: 'from' test | testlist_star_expr diff --git a/Lib/lib2to3/PatternGrammar.txt b/Lib/lib2to3/PatternGrammar.txt deleted file mode 100644 index 36bf8148273bd7..00000000000000 --- a/Lib/lib2to3/PatternGrammar.txt +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -# A grammar to describe tree matching patterns. -# Not shown here: -# - 'TOKEN' stands for any token (leaf node) -# - 'any' stands for any node (leaf or interior) -# With 'any' we can still specify the sub-structure. - -# The start symbol is 'Matcher'. - -Matcher: Alternatives ENDMARKER - -Alternatives: Alternative ('|' Alternative)* - -Alternative: (Unit | NegatedUnit)+ - -Unit: [NAME '='] ( STRING [Repeater] - | NAME [Details] [Repeater] - | '(' Alternatives ')' [Repeater] - | '[' Alternatives ']' - ) - -NegatedUnit: 'not' (STRING | NAME [Details] | '(' Alternatives ')') - -Repeater: '*' | '+' | '{' NUMBER [',' NUMBER] '}' - -Details: '<' Alternatives '>' diff --git a/Lib/lib2to3/__init__.py b/Lib/lib2to3/__init__.py deleted file mode 100644 index 177405c8090d3e..00000000000000 --- a/Lib/lib2to3/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -import warnings - - -warnings.warn( - "lib2to3 package is deprecated and may not be able to parse Python 3.10+", - DeprecationWarning, - stacklevel=2, -) diff --git a/Lib/lib2to3/__main__.py b/Lib/lib2to3/__main__.py deleted file mode 100644 index 80688baf27abfc..00000000000000 --- a/Lib/lib2to3/__main__.py +++ /dev/null @@ -1,4 +0,0 @@ -import sys -from .main import main - -sys.exit(main("lib2to3.fixes")) diff --git a/Lib/lib2to3/btm_matcher.py b/Lib/lib2to3/btm_matcher.py deleted file mode 100644 index 3b78868038bda0..00000000000000 --- a/Lib/lib2to3/btm_matcher.py +++ /dev/null @@ -1,163 +0,0 @@ -"""A bottom-up tree matching algorithm implementation meant to speed -up 2to3's matching process. After the tree patterns are reduced to -their rarest linear path, a linear Aho-Corasick automaton is -created. The linear automaton traverses the linear paths from the -leaves to the root of the AST and returns a set of nodes for further -matching. This reduces significantly the number of candidate nodes.""" - -__author__ = "George Boutsioukis " - -import logging -import itertools -from collections import defaultdict - -from . import pytree -from .btm_utils import reduce_tree - -class BMNode(object): - """Class for a node of the Aho-Corasick automaton used in matching""" - count = itertools.count() - def __init__(self): - self.transition_table = {} - self.fixers = [] - self.id = next(BMNode.count) - self.content = '' - -class BottomMatcher(object): - """The main matcher class. After instantiating the patterns should - be added using the add_fixer method""" - - def __init__(self): - self.match = set() - self.root = BMNode() - self.nodes = [self.root] - self.fixers = [] - self.logger = logging.getLogger("RefactoringTool") - - def add_fixer(self, fixer): - """Reduces a fixer's pattern tree to a linear path and adds it - to the matcher(a common Aho-Corasick automaton). The fixer is - appended on the matching states and called when they are - reached""" - self.fixers.append(fixer) - tree = reduce_tree(fixer.pattern_tree) - linear = tree.get_linear_subpattern() - match_nodes = self.add(linear, start=self.root) - for match_node in match_nodes: - match_node.fixers.append(fixer) - - def add(self, pattern, start): - "Recursively adds a linear pattern to the AC automaton" - #print("adding pattern", pattern, "to", start) - if not pattern: - #print("empty pattern") - return [start] - if isinstance(pattern[0], tuple): - #alternatives - #print("alternatives") - match_nodes = [] - for alternative in pattern[0]: - #add all alternatives, and add the rest of the pattern - #to each end node - end_nodes = self.add(alternative, start=start) - for end in end_nodes: - match_nodes.extend(self.add(pattern[1:], end)) - return match_nodes - else: - #single token - #not last - if pattern[0] not in start.transition_table: - #transition did not exist, create new - next_node = BMNode() - start.transition_table[pattern[0]] = next_node - else: - #transition exists already, follow - next_node = start.transition_table[pattern[0]] - - if pattern[1:]: - end_nodes = self.add(pattern[1:], start=next_node) - else: - end_nodes = [next_node] - return end_nodes - - def run(self, leaves): - """The main interface with the bottom matcher. The tree is - traversed from the bottom using the constructed - automaton. Nodes are only checked once as the tree is - retraversed. When the automaton fails, we give it one more - shot(in case the above tree matches as a whole with the - rejected leaf), then we break for the next leaf. There is the - special case of multiple arguments(see code comments) where we - recheck the nodes - - Args: - The leaves of the AST tree to be matched - - Returns: - A dictionary of node matches with fixers as the keys - """ - current_ac_node = self.root - results = defaultdict(list) - for leaf in leaves: - current_ast_node = leaf - while current_ast_node: - current_ast_node.was_checked = True - for child in current_ast_node.children: - # multiple statements, recheck - if isinstance(child, pytree.Leaf) and child.value == ";": - current_ast_node.was_checked = False - break - if current_ast_node.type == 1: - #name - node_token = current_ast_node.value - else: - node_token = current_ast_node.type - - if node_token in current_ac_node.transition_table: - #token matches - current_ac_node = current_ac_node.transition_table[node_token] - for fixer in current_ac_node.fixers: - results[fixer].append(current_ast_node) - else: - #matching failed, reset automaton - current_ac_node = self.root - if (current_ast_node.parent is not None - and current_ast_node.parent.was_checked): - #the rest of the tree upwards has been checked, next leaf - break - - #recheck the rejected node once from the root - if node_token in current_ac_node.transition_table: - #token matches - current_ac_node = current_ac_node.transition_table[node_token] - for fixer in current_ac_node.fixers: - results[fixer].append(current_ast_node) - - current_ast_node = current_ast_node.parent - return results - - def print_ac(self): - "Prints a graphviz diagram of the BM automaton(for debugging)" - print("digraph g{") - def print_node(node): - for subnode_key in node.transition_table.keys(): - subnode = node.transition_table[subnode_key] - print("%d -> %d [label=%s] //%s" % - (node.id, subnode.id, type_repr(subnode_key), str(subnode.fixers))) - if subnode_key == 1: - print(subnode.content) - print_node(subnode) - print_node(self.root) - print("}") - -# taken from pytree.py for debugging; only used by print_ac -_type_reprs = {} -def type_repr(type_num): - global _type_reprs - if not _type_reprs: - from .pygram import python_symbols - # printing tokens is possible but not as useful - # from .pgen2 import token // token.__dict__.items(): - for name, val in python_symbols.__dict__.items(): - if type(val) == int: _type_reprs[val] = name - return _type_reprs.setdefault(type_num, type_num) diff --git a/Lib/lib2to3/btm_utils.py b/Lib/lib2to3/btm_utils.py deleted file mode 100644 index b61afdba693071..00000000000000 --- a/Lib/lib2to3/btm_utils.py +++ /dev/null @@ -1,280 +0,0 @@ -"Utility functions used by the btm_matcher module" - -from . import pytree -from .pgen2 import grammar, token -from .pygram import pattern_symbols, python_symbols - -syms = pattern_symbols -pysyms = python_symbols -tokens = grammar.opmap -token_labels = token - -TYPE_ANY = -1 -TYPE_ALTERNATIVES = -2 -TYPE_GROUP = -3 - -class MinNode(object): - """This class serves as an intermediate representation of the - pattern tree during the conversion to sets of leaf-to-root - subpatterns""" - - def __init__(self, type=None, name=None): - self.type = type - self.name = name - self.children = [] - self.leaf = False - self.parent = None - self.alternatives = [] - self.group = [] - - def __repr__(self): - return str(self.type) + ' ' + str(self.name) - - def leaf_to_root(self): - """Internal method. Returns a characteristic path of the - pattern tree. This method must be run for all leaves until the - linear subpatterns are merged into a single""" - node = self - subp = [] - while node: - if node.type == TYPE_ALTERNATIVES: - node.alternatives.append(subp) - if len(node.alternatives) == len(node.children): - #last alternative - subp = [tuple(node.alternatives)] - node.alternatives = [] - node = node.parent - continue - else: - node = node.parent - subp = None - break - - if node.type == TYPE_GROUP: - node.group.append(subp) - #probably should check the number of leaves - if len(node.group) == len(node.children): - subp = get_characteristic_subpattern(node.group) - node.group = [] - node = node.parent - continue - else: - node = node.parent - subp = None - break - - if node.type == token_labels.NAME and node.name: - #in case of type=name, use the name instead - subp.append(node.name) - else: - subp.append(node.type) - - node = node.parent - return subp - - def get_linear_subpattern(self): - """Drives the leaf_to_root method. The reason that - leaf_to_root must be run multiple times is because we need to - reject 'group' matches; for example the alternative form - (a | b c) creates a group [b c] that needs to be matched. Since - matching multiple linear patterns overcomes the automaton's - capabilities, leaf_to_root merges each group into a single - choice based on 'characteristic'ity, - - i.e. (a|b c) -> (a|b) if b more characteristic than c - - Returns: The most 'characteristic'(as defined by - get_characteristic_subpattern) path for the compiled pattern - tree. - """ - - for l in self.leaves(): - subp = l.leaf_to_root() - if subp: - return subp - - def leaves(self): - "Generator that returns the leaves of the tree" - for child in self.children: - yield from child.leaves() - if not self.children: - yield self - -def reduce_tree(node, parent=None): - """ - Internal function. Reduces a compiled pattern tree to an - intermediate representation suitable for feeding the - automaton. This also trims off any optional pattern elements(like - [a], a*). - """ - - new_node = None - #switch on the node type - if node.type == syms.Matcher: - #skip - node = node.children[0] - - if node.type == syms.Alternatives : - #2 cases - if len(node.children) <= 2: - #just a single 'Alternative', skip this node - new_node = reduce_tree(node.children[0], parent) - else: - #real alternatives - new_node = MinNode(type=TYPE_ALTERNATIVES) - #skip odd children('|' tokens) - for child in node.children: - if node.children.index(child)%2: - continue - reduced = reduce_tree(child, new_node) - if reduced is not None: - new_node.children.append(reduced) - elif node.type == syms.Alternative: - if len(node.children) > 1: - - new_node = MinNode(type=TYPE_GROUP) - for child in node.children: - reduced = reduce_tree(child, new_node) - if reduced: - new_node.children.append(reduced) - if not new_node.children: - # delete the group if all of the children were reduced to None - new_node = None - - else: - new_node = reduce_tree(node.children[0], parent) - - elif node.type == syms.Unit: - if (isinstance(node.children[0], pytree.Leaf) and - node.children[0].value == '('): - #skip parentheses - return reduce_tree(node.children[1], parent) - if ((isinstance(node.children[0], pytree.Leaf) and - node.children[0].value == '[') - or - (len(node.children)>1 and - hasattr(node.children[1], "value") and - node.children[1].value == '[')): - #skip whole unit if its optional - return None - - leaf = True - details_node = None - alternatives_node = None - has_repeater = False - repeater_node = None - has_variable_name = False - - for child in node.children: - if child.type == syms.Details: - leaf = False - details_node = child - elif child.type == syms.Repeater: - has_repeater = True - repeater_node = child - elif child.type == syms.Alternatives: - alternatives_node = child - if hasattr(child, 'value') and child.value == '=': # variable name - has_variable_name = True - - #skip variable name - if has_variable_name: - #skip variable name, '=' - name_leaf = node.children[2] - if hasattr(name_leaf, 'value') and name_leaf.value == '(': - # skip parenthesis - name_leaf = node.children[3] - else: - name_leaf = node.children[0] - - #set node type - if name_leaf.type == token_labels.NAME: - #(python) non-name or wildcard - if name_leaf.value == 'any': - new_node = MinNode(type=TYPE_ANY) - else: - if hasattr(token_labels, name_leaf.value): - new_node = MinNode(type=getattr(token_labels, name_leaf.value)) - else: - new_node = MinNode(type=getattr(pysyms, name_leaf.value)) - - elif name_leaf.type == token_labels.STRING: - #(python) name or character; remove the apostrophes from - #the string value - name = name_leaf.value.strip("'") - if name in tokens: - new_node = MinNode(type=tokens[name]) - else: - new_node = MinNode(type=token_labels.NAME, name=name) - elif name_leaf.type == syms.Alternatives: - new_node = reduce_tree(alternatives_node, parent) - - #handle repeaters - if has_repeater: - if repeater_node.children[0].value == '*': - #reduce to None - new_node = None - elif repeater_node.children[0].value == '+': - #reduce to a single occurrence i.e. do nothing - pass - else: - #TODO: handle {min, max} repeaters - raise NotImplementedError - - #add children - if details_node and new_node is not None: - for child in details_node.children[1:-1]: - #skip '<', '>' markers - reduced = reduce_tree(child, new_node) - if reduced is not None: - new_node.children.append(reduced) - if new_node: - new_node.parent = parent - return new_node - - -def get_characteristic_subpattern(subpatterns): - """Picks the most characteristic from a list of linear patterns - Current order used is: - names > common_names > common_chars - """ - if not isinstance(subpatterns, list): - return subpatterns - if len(subpatterns)==1: - return subpatterns[0] - - # first pick out the ones containing variable names - subpatterns_with_names = [] - subpatterns_with_common_names = [] - common_names = ['in', 'for', 'if' , 'not', 'None'] - subpatterns_with_common_chars = [] - common_chars = "[]().,:" - for subpattern in subpatterns: - if any(rec_test(subpattern, lambda x: type(x) is str)): - if any(rec_test(subpattern, - lambda x: isinstance(x, str) and x in common_chars)): - subpatterns_with_common_chars.append(subpattern) - elif any(rec_test(subpattern, - lambda x: isinstance(x, str) and x in common_names)): - subpatterns_with_common_names.append(subpattern) - - else: - subpatterns_with_names.append(subpattern) - - if subpatterns_with_names: - subpatterns = subpatterns_with_names - elif subpatterns_with_common_names: - subpatterns = subpatterns_with_common_names - elif subpatterns_with_common_chars: - subpatterns = subpatterns_with_common_chars - # of the remaining subpatterns pick out the longest one - return max(subpatterns, key=len) - -def rec_test(sequence, test_func): - """Tests test_func on all items of sequence and items of included - sub-iterables""" - for x in sequence: - if isinstance(x, (list, tuple)): - yield from rec_test(x, test_func) - else: - yield test_func(x) diff --git a/Lib/lib2to3/fixer_base.py b/Lib/lib2to3/fixer_base.py deleted file mode 100644 index df581a4deab9e9..00000000000000 --- a/Lib/lib2to3/fixer_base.py +++ /dev/null @@ -1,186 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Base class for fixers (optional, but recommended).""" - -# Python imports -import itertools - -# Local imports -from .patcomp import PatternCompiler -from . import pygram -from .fixer_util import does_tree_import - -class BaseFix(object): - - """Optional base class for fixers. - - The subclass name must be FixFooBar where FooBar is the result of - removing underscores and capitalizing the words of the fix name. - For example, the class name for a fixer named 'has_key' should be - FixHasKey. - """ - - PATTERN = None # Most subclasses should override with a string literal - pattern = None # Compiled pattern, set by compile_pattern() - pattern_tree = None # Tree representation of the pattern - options = None # Options object passed to initializer - filename = None # The filename (set by set_filename) - numbers = itertools.count(1) # For new_name() - used_names = set() # A set of all used NAMEs - order = "post" # Does the fixer prefer pre- or post-order traversal - explicit = False # Is this ignored by refactor.py -f all? - run_order = 5 # Fixers will be sorted by run order before execution - # Lower numbers will be run first. - _accept_type = None # [Advanced and not public] This tells RefactoringTool - # which node type to accept when there's not a pattern. - - keep_line_order = False # For the bottom matcher: match with the - # original line order - BM_compatible = False # Compatibility with the bottom matching - # module; every fixer should set this - # manually - - # Shortcut for access to Python grammar symbols - syms = pygram.python_symbols - - def __init__(self, options, log): - """Initializer. Subclass may override. - - Args: - options: a dict containing the options passed to RefactoringTool - that could be used to customize the fixer through the command line. - log: a list to append warnings and other messages to. - """ - self.options = options - self.log = log - self.compile_pattern() - - def compile_pattern(self): - """Compiles self.PATTERN into self.pattern. - - Subclass may override if it doesn't want to use - self.{pattern,PATTERN} in .match(). - """ - if self.PATTERN is not None: - PC = PatternCompiler() - self.pattern, self.pattern_tree = PC.compile_pattern(self.PATTERN, - with_tree=True) - - def set_filename(self, filename): - """Set the filename. - - The main refactoring tool should call this. - """ - self.filename = filename - - def match(self, node): - """Returns match for a given parse tree node. - - Should return a true or false object (not necessarily a bool). - It may return a non-empty dict of matching sub-nodes as - returned by a matching pattern. - - Subclass may override. - """ - results = {"node": node} - return self.pattern.match(node, results) and results - - def transform(self, node, results): - """Returns the transformation for a given parse tree node. - - Args: - node: the root of the parse tree that matched the fixer. - results: a dict mapping symbolic names to part of the match. - - Returns: - None, or a node that is a modified copy of the - argument node. The node argument may also be modified in-place to - effect the same change. - - Subclass *must* override. - """ - raise NotImplementedError() - - def new_name(self, template="xxx_todo_changeme"): - """Return a string suitable for use as an identifier - - The new name is guaranteed not to conflict with other identifiers. - """ - name = template - while name in self.used_names: - name = template + str(next(self.numbers)) - self.used_names.add(name) - return name - - def log_message(self, message): - if self.first_log: - self.first_log = False - self.log.append("### In file %s ###" % self.filename) - self.log.append(message) - - def cannot_convert(self, node, reason=None): - """Warn the user that a given chunk of code is not valid Python 3, - but that it cannot be converted automatically. - - First argument is the top-level node for the code in question. - Optional second argument is why it can't be converted. - """ - lineno = node.get_lineno() - for_output = node.clone() - for_output.prefix = "" - msg = "Line %d: could not convert: %s" - self.log_message(msg % (lineno, for_output)) - if reason: - self.log_message(reason) - - def warning(self, node, reason): - """Used for warning the user about possible uncertainty in the - translation. - - First argument is the top-level node for the code in question. - Optional second argument is why it can't be converted. - """ - lineno = node.get_lineno() - self.log_message("Line %d: %s" % (lineno, reason)) - - def start_tree(self, tree, filename): - """Some fixers need to maintain tree-wide state. - This method is called once, at the start of tree fix-up. - - tree - the root node of the tree to be processed. - filename - the name of the file the tree came from. - """ - self.used_names = tree.used_names - self.set_filename(filename) - self.numbers = itertools.count(1) - self.first_log = True - - def finish_tree(self, tree, filename): - """Some fixers need to maintain tree-wide state. - This method is called once, at the conclusion of tree fix-up. - - tree - the root node of the tree to be processed. - filename - the name of the file the tree came from. - """ - pass - - -class ConditionalFix(BaseFix): - """ Base class for fixers which not execute if an import is found. """ - - # This is the name of the import which, if found, will cause the test to be skipped - skip_on = None - - def start_tree(self, *args): - super(ConditionalFix, self).start_tree(*args) - self._should_skip = None - - def should_skip(self, node): - if self._should_skip is not None: - return self._should_skip - pkg = self.skip_on.split(".") - name = pkg[-1] - pkg = ".".join(pkg[:-1]) - self._should_skip = does_tree_import(pkg, name, node) - return self._should_skip diff --git a/Lib/lib2to3/fixer_util.py b/Lib/lib2to3/fixer_util.py deleted file mode 100644 index c2a3a47f503286..00000000000000 --- a/Lib/lib2to3/fixer_util.py +++ /dev/null @@ -1,453 +0,0 @@ -"""Utility functions, node construction macros, etc.""" -# Author: Collin Winter - -# Local imports -from .pgen2 import token -from .pytree import Leaf, Node -from .pygram import python_symbols as syms -from . import patcomp - - -########################################################### -### Common node-construction "macros" -########################################################### - -def KeywordArg(keyword, value): - return Node(syms.argument, - [keyword, Leaf(token.EQUAL, "="), value]) - -def LParen(): - return Leaf(token.LPAR, "(") - -def RParen(): - return Leaf(token.RPAR, ")") - -def Assign(target, source): - """Build an assignment statement""" - if not isinstance(target, list): - target = [target] - if not isinstance(source, list): - source.prefix = " " - source = [source] - - return Node(syms.atom, - target + [Leaf(token.EQUAL, "=", prefix=" ")] + source) - -def Name(name, prefix=None): - """Return a NAME leaf""" - return Leaf(token.NAME, name, prefix=prefix) - -def Attr(obj, attr): - """A node tuple for obj.attr""" - return [obj, Node(syms.trailer, [Dot(), attr])] - -def Comma(): - """A comma leaf""" - return Leaf(token.COMMA, ",") - -def Dot(): - """A period (.) leaf""" - return Leaf(token.DOT, ".") - -def ArgList(args, lparen=LParen(), rparen=RParen()): - """A parenthesised argument list, used by Call()""" - node = Node(syms.trailer, [lparen.clone(), rparen.clone()]) - if args: - node.insert_child(1, Node(syms.arglist, args)) - return node - -def Call(func_name, args=None, prefix=None): - """A function call""" - node = Node(syms.power, [func_name, ArgList(args)]) - if prefix is not None: - node.prefix = prefix - return node - -def Newline(): - """A newline literal""" - return Leaf(token.NEWLINE, "\n") - -def BlankLine(): - """A blank line""" - return Leaf(token.NEWLINE, "") - -def Number(n, prefix=None): - return Leaf(token.NUMBER, n, prefix=prefix) - -def Subscript(index_node): - """A numeric or string subscript""" - return Node(syms.trailer, [Leaf(token.LBRACE, "["), - index_node, - Leaf(token.RBRACE, "]")]) - -def String(string, prefix=None): - """A string leaf""" - return Leaf(token.STRING, string, prefix=prefix) - -def ListComp(xp, fp, it, test=None): - """A list comprehension of the form [xp for fp in it if test]. - - If test is None, the "if test" part is omitted. - """ - xp.prefix = "" - fp.prefix = " " - it.prefix = " " - for_leaf = Leaf(token.NAME, "for") - for_leaf.prefix = " " - in_leaf = Leaf(token.NAME, "in") - in_leaf.prefix = " " - inner_args = [for_leaf, fp, in_leaf, it] - if test: - test.prefix = " " - if_leaf = Leaf(token.NAME, "if") - if_leaf.prefix = " " - inner_args.append(Node(syms.comp_if, [if_leaf, test])) - inner = Node(syms.listmaker, [xp, Node(syms.comp_for, inner_args)]) - return Node(syms.atom, - [Leaf(token.LBRACE, "["), - inner, - Leaf(token.RBRACE, "]")]) - -def FromImport(package_name, name_leafs): - """ Return an import statement in the form: - from package import name_leafs""" - # XXX: May not handle dotted imports properly (eg, package_name='foo.bar') - #assert package_name == '.' or '.' not in package_name, "FromImport has "\ - # "not been tested with dotted package names -- use at your own "\ - # "peril!" - - for leaf in name_leafs: - # Pull the leaves out of their old tree - leaf.remove() - - children = [Leaf(token.NAME, "from"), - Leaf(token.NAME, package_name, prefix=" "), - Leaf(token.NAME, "import", prefix=" "), - Node(syms.import_as_names, name_leafs)] - imp = Node(syms.import_from, children) - return imp - -def ImportAndCall(node, results, names): - """Returns an import statement and calls a method - of the module: - - import module - module.name()""" - obj = results["obj"].clone() - if obj.type == syms.arglist: - newarglist = obj.clone() - else: - newarglist = Node(syms.arglist, [obj.clone()]) - after = results["after"] - if after: - after = [n.clone() for n in after] - new = Node(syms.power, - Attr(Name(names[0]), Name(names[1])) + - [Node(syms.trailer, - [results["lpar"].clone(), - newarglist, - results["rpar"].clone()])] + after) - new.prefix = node.prefix - return new - - -########################################################### -### Determine whether a node represents a given literal -########################################################### - -def is_tuple(node): - """Does the node represent a tuple literal?""" - if isinstance(node, Node) and node.children == [LParen(), RParen()]: - return True - return (isinstance(node, Node) - and len(node.children) == 3 - and isinstance(node.children[0], Leaf) - and isinstance(node.children[1], Node) - and isinstance(node.children[2], Leaf) - and node.children[0].value == "(" - and node.children[2].value == ")") - -def is_list(node): - """Does the node represent a list literal?""" - return (isinstance(node, Node) - and len(node.children) > 1 - and isinstance(node.children[0], Leaf) - and isinstance(node.children[-1], Leaf) - and node.children[0].value == "[" - and node.children[-1].value == "]") - - -########################################################### -### Misc -########################################################### - -def parenthesize(node): - return Node(syms.atom, [LParen(), node, RParen()]) - - -consuming_calls = {"sorted", "list", "set", "any", "all", "tuple", "sum", - "min", "max", "enumerate"} - -def attr_chain(obj, attr): - """Follow an attribute chain. - - If you have a chain of objects where a.foo -> b, b.foo-> c, etc, - use this to iterate over all objects in the chain. Iteration is - terminated by getattr(x, attr) is None. - - Args: - obj: the starting object - attr: the name of the chaining attribute - - Yields: - Each successive object in the chain. - """ - next = getattr(obj, attr) - while next: - yield next - next = getattr(next, attr) - -p0 = """for_stmt< 'for' any 'in' node=any ':' any* > - | comp_for< 'for' any 'in' node=any any* > - """ -p1 = """ -power< - ( 'iter' | 'list' | 'tuple' | 'sorted' | 'set' | 'sum' | - 'any' | 'all' | 'enumerate' | (any* trailer< '.' 'join' >) ) - trailer< '(' node=any ')' > - any* -> -""" -p2 = """ -power< - ( 'sorted' | 'enumerate' ) - trailer< '(' arglist ')' > - any* -> -""" -pats_built = False -def in_special_context(node): - """ Returns true if node is in an environment where all that is required - of it is being iterable (ie, it doesn't matter if it returns a list - or an iterator). - See test_map_nochange in test_fixers.py for some examples and tests. - """ - global p0, p1, p2, pats_built - if not pats_built: - p0 = patcomp.compile_pattern(p0) - p1 = patcomp.compile_pattern(p1) - p2 = patcomp.compile_pattern(p2) - pats_built = True - patterns = [p0, p1, p2] - for pattern, parent in zip(patterns, attr_chain(node, "parent")): - results = {} - if pattern.match(parent, results) and results["node"] is node: - return True - return False - -def is_probably_builtin(node): - """ - Check that something isn't an attribute or function name etc. - """ - prev = node.prev_sibling - if prev is not None and prev.type == token.DOT: - # Attribute lookup. - return False - parent = node.parent - if parent.type in (syms.funcdef, syms.classdef): - return False - if parent.type == syms.expr_stmt and parent.children[0] is node: - # Assignment. - return False - if parent.type == syms.parameters or \ - (parent.type == syms.typedargslist and ( - (prev is not None and prev.type == token.COMMA) or - parent.children[0] is node - )): - # The name of an argument. - return False - return True - -def find_indentation(node): - """Find the indentation of *node*.""" - while node is not None: - if node.type == syms.suite and len(node.children) > 2: - indent = node.children[1] - if indent.type == token.INDENT: - return indent.value - node = node.parent - return "" - -########################################################### -### The following functions are to find bindings in a suite -########################################################### - -def make_suite(node): - if node.type == syms.suite: - return node - node = node.clone() - parent, node.parent = node.parent, None - suite = Node(syms.suite, [node]) - suite.parent = parent - return suite - -def find_root(node): - """Find the top level namespace.""" - # Scamper up to the top level namespace - while node.type != syms.file_input: - node = node.parent - if not node: - raise ValueError("root found before file_input node was found.") - return node - -def does_tree_import(package, name, node): - """ Returns true if name is imported from package at the - top level of the tree which node belongs to. - To cover the case of an import like 'import foo', use - None for the package and 'foo' for the name. """ - binding = find_binding(name, find_root(node), package) - return bool(binding) - -def is_import(node): - """Returns true if the node is an import statement.""" - return node.type in (syms.import_name, syms.import_from) - -def touch_import(package, name, node): - """ Works like `does_tree_import` but adds an import statement - if it was not imported. """ - def is_import_stmt(node): - return (node.type == syms.simple_stmt and node.children and - is_import(node.children[0])) - - root = find_root(node) - - if does_tree_import(package, name, root): - return - - # figure out where to insert the new import. First try to find - # the first import and then skip to the last one. - insert_pos = offset = 0 - for idx, node in enumerate(root.children): - if not is_import_stmt(node): - continue - for offset, node2 in enumerate(root.children[idx:]): - if not is_import_stmt(node2): - break - insert_pos = idx + offset - break - - # if there are no imports where we can insert, find the docstring. - # if that also fails, we stick to the beginning of the file - if insert_pos == 0: - for idx, node in enumerate(root.children): - if (node.type == syms.simple_stmt and node.children and - node.children[0].type == token.STRING): - insert_pos = idx + 1 - break - - if package is None: - import_ = Node(syms.import_name, [ - Leaf(token.NAME, "import"), - Leaf(token.NAME, name, prefix=" ") - ]) - else: - import_ = FromImport(package, [Leaf(token.NAME, name, prefix=" ")]) - - children = [import_, Newline()] - root.insert_child(insert_pos, Node(syms.simple_stmt, children)) - - -_def_syms = {syms.classdef, syms.funcdef} -def find_binding(name, node, package=None): - """ Returns the node which binds variable name, otherwise None. - If optional argument package is supplied, only imports will - be returned. - See test cases for examples.""" - for child in node.children: - ret = None - if child.type == syms.for_stmt: - if _find(name, child.children[1]): - return child - n = find_binding(name, make_suite(child.children[-1]), package) - if n: ret = n - elif child.type in (syms.if_stmt, syms.while_stmt): - n = find_binding(name, make_suite(child.children[-1]), package) - if n: ret = n - elif child.type == syms.try_stmt: - n = find_binding(name, make_suite(child.children[2]), package) - if n: - ret = n - else: - for i, kid in enumerate(child.children[3:]): - if kid.type == token.COLON and kid.value == ":": - # i+3 is the colon, i+4 is the suite - n = find_binding(name, make_suite(child.children[i+4]), package) - if n: ret = n - elif child.type in _def_syms and child.children[1].value == name: - ret = child - elif _is_import_binding(child, name, package): - ret = child - elif child.type == syms.simple_stmt: - ret = find_binding(name, child, package) - elif child.type == syms.expr_stmt: - if _find(name, child.children[0]): - ret = child - - if ret: - if not package: - return ret - if is_import(ret): - return ret - return None - -_block_syms = {syms.funcdef, syms.classdef, syms.trailer} -def _find(name, node): - nodes = [node] - while nodes: - node = nodes.pop() - if node.type > 256 and node.type not in _block_syms: - nodes.extend(node.children) - elif node.type == token.NAME and node.value == name: - return node - return None - -def _is_import_binding(node, name, package=None): - """ Will return node if node will import name, or node - will import * from package. None is returned otherwise. - See test cases for examples. """ - - if node.type == syms.import_name and not package: - imp = node.children[1] - if imp.type == syms.dotted_as_names: - for child in imp.children: - if child.type == syms.dotted_as_name: - if child.children[2].value == name: - return node - elif child.type == token.NAME and child.value == name: - return node - elif imp.type == syms.dotted_as_name: - last = imp.children[-1] - if last.type == token.NAME and last.value == name: - return node - elif imp.type == token.NAME and imp.value == name: - return node - elif node.type == syms.import_from: - # str(...) is used to make life easier here, because - # from a.b import parses to ['import', ['a', '.', 'b'], ...] - if package and str(node.children[1]).strip() != package: - return None - n = node.children[3] - if package and _find("as", n): - # See test_from_import_as for explanation - return None - elif n.type == syms.import_as_names and _find(name, n): - return node - elif n.type == syms.import_as_name: - child = n.children[2] - if child.type == token.NAME and child.value == name: - return node - elif n.type == token.NAME and n.value == name: - return node - elif package and n.type == token.STAR: - return node - return None diff --git a/Lib/lib2to3/fixes/__init__.py b/Lib/lib2to3/fixes/__init__.py deleted file mode 100644 index b93054b3ecf3a5..00000000000000 --- a/Lib/lib2to3/fixes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Dummy file to make this directory a package. diff --git a/Lib/lib2to3/fixes/fix_apply.py b/Lib/lib2to3/fixes/fix_apply.py deleted file mode 100644 index 6408582c426477..00000000000000 --- a/Lib/lib2to3/fixes/fix_apply.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for apply(). - -This converts apply(func, v, k) into (func)(*v, **k).""" - -# Local imports -from .. import pytree -from ..pgen2 import token -from .. import fixer_base -from ..fixer_util import Call, Comma, parenthesize - -class FixApply(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - power< 'apply' - trailer< - '(' - arglist< - (not argument - ')' - > - > - """ - - def transform(self, node, results): - syms = self.syms - assert results - func = results["func"] - args = results["args"] - kwds = results.get("kwds") - # I feel like we should be able to express this logic in the - # PATTERN above but I don't know how to do it so... - if args: - if (args.type == self.syms.argument and - args.children[0].value in {'**', '*'}): - return # Make no change. - if kwds and (kwds.type == self.syms.argument and - kwds.children[0].value == '**'): - return # Make no change. - prefix = node.prefix - func = func.clone() - if (func.type not in (token.NAME, syms.atom) and - (func.type != syms.power or - func.children[-2].type == token.DOUBLESTAR)): - # Need to parenthesize - func = parenthesize(func) - func.prefix = "" - args = args.clone() - args.prefix = "" - if kwds is not None: - kwds = kwds.clone() - kwds.prefix = "" - l_newargs = [pytree.Leaf(token.STAR, "*"), args] - if kwds is not None: - l_newargs.extend([Comma(), - pytree.Leaf(token.DOUBLESTAR, "**"), - kwds]) - l_newargs[-2].prefix = " " # that's the ** token - # XXX Sometimes we could be cleverer, e.g. apply(f, (x, y) + t) - # can be translated into f(x, y, *t) instead of f(*(x, y) + t) - #new = pytree.Node(syms.power, (func, ArgList(l_newargs))) - return Call(func, l_newargs, prefix=prefix) diff --git a/Lib/lib2to3/fixes/fix_asserts.py b/Lib/lib2to3/fixes/fix_asserts.py deleted file mode 100644 index 5bcec885f52cbf..00000000000000 --- a/Lib/lib2to3/fixes/fix_asserts.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Fixer that replaces deprecated unittest method names.""" - -# Author: Ezio Melotti - -from ..fixer_base import BaseFix -from ..fixer_util import Name - -NAMES = dict( - assert_="assertTrue", - assertEquals="assertEqual", - assertNotEquals="assertNotEqual", - assertAlmostEquals="assertAlmostEqual", - assertNotAlmostEquals="assertNotAlmostEqual", - assertRegexpMatches="assertRegex", - assertRaisesRegexp="assertRaisesRegex", - failUnlessEqual="assertEqual", - failIfEqual="assertNotEqual", - failUnlessAlmostEqual="assertAlmostEqual", - failIfAlmostEqual="assertNotAlmostEqual", - failUnless="assertTrue", - failUnlessRaises="assertRaises", - failIf="assertFalse", -) - - -class FixAsserts(BaseFix): - - PATTERN = """ - power< any+ trailer< '.' meth=(%s)> any* > - """ % '|'.join(map(repr, NAMES)) - - def transform(self, node, results): - name = results["meth"][0] - name.replace(Name(NAMES[str(name)], prefix=name.prefix)) diff --git a/Lib/lib2to3/fixes/fix_basestring.py b/Lib/lib2to3/fixes/fix_basestring.py deleted file mode 100644 index 5fe69a0f03b1b8..00000000000000 --- a/Lib/lib2to3/fixes/fix_basestring.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Fixer for basestring -> str.""" -# Author: Christian Heimes - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - -class FixBasestring(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = "'basestring'" - - def transform(self, node, results): - return Name("str", prefix=node.prefix) diff --git a/Lib/lib2to3/fixes/fix_buffer.py b/Lib/lib2to3/fixes/fix_buffer.py deleted file mode 100644 index f9a1958ad3b93e..00000000000000 --- a/Lib/lib2to3/fixes/fix_buffer.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2007 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer that changes buffer(...) into memoryview(...).""" - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - - -class FixBuffer(fixer_base.BaseFix): - BM_compatible = True - - explicit = True # The user must ask for this fixer - - PATTERN = """ - power< name='buffer' trailer< '(' [any] ')' > any* > - """ - - def transform(self, node, results): - name = results["name"] - name.replace(Name("memoryview", prefix=name.prefix)) diff --git a/Lib/lib2to3/fixes/fix_dict.py b/Lib/lib2to3/fixes/fix_dict.py deleted file mode 100644 index d3655c9f1b2d9b..00000000000000 --- a/Lib/lib2to3/fixes/fix_dict.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright 2007 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for dict methods. - -d.keys() -> list(d.keys()) -d.items() -> list(d.items()) -d.values() -> list(d.values()) - -d.iterkeys() -> iter(d.keys()) -d.iteritems() -> iter(d.items()) -d.itervalues() -> iter(d.values()) - -d.viewkeys() -> d.keys() -d.viewitems() -> d.items() -d.viewvalues() -> d.values() - -Except in certain very specific contexts: the iter() can be dropped -when the context is list(), sorted(), iter() or for...in; the list() -can be dropped when the context is list() or sorted() (but not iter() -or for...in!). Special contexts that apply to both: list(), sorted(), tuple() -set(), any(), all(), sum(). - -Note: iter(d.keys()) could be written as iter(d) but since the -original d.iterkeys() was also redundant we don't fix this. And there -are (rare) contexts where it makes a difference (e.g. when passing it -as an argument to a function that introspects the argument). -""" - -# Local imports -from .. import pytree -from .. import patcomp -from .. import fixer_base -from ..fixer_util import Name, Call, Dot -from .. import fixer_util - - -iter_exempt = fixer_util.consuming_calls | {"iter"} - - -class FixDict(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - power< head=any+ - trailer< '.' method=('keys'|'items'|'values'| - 'iterkeys'|'iteritems'|'itervalues'| - 'viewkeys'|'viewitems'|'viewvalues') > - parens=trailer< '(' ')' > - tail=any* - > - """ - - def transform(self, node, results): - head = results["head"] - method = results["method"][0] # Extract node for method name - tail = results["tail"] - syms = self.syms - method_name = method.value - isiter = method_name.startswith("iter") - isview = method_name.startswith("view") - if isiter or isview: - method_name = method_name[4:] - assert method_name in ("keys", "items", "values"), repr(method) - head = [n.clone() for n in head] - tail = [n.clone() for n in tail] - special = not tail and self.in_special_context(node, isiter) - args = head + [pytree.Node(syms.trailer, - [Dot(), - Name(method_name, - prefix=method.prefix)]), - results["parens"].clone()] - new = pytree.Node(syms.power, args) - if not (special or isview): - new.prefix = "" - new = Call(Name("iter" if isiter else "list"), [new]) - if tail: - new = pytree.Node(syms.power, [new] + tail) - new.prefix = node.prefix - return new - - P1 = "power< func=NAME trailer< '(' node=any ')' > any* >" - p1 = patcomp.compile_pattern(P1) - - P2 = """for_stmt< 'for' any 'in' node=any ':' any* > - | comp_for< 'for' any 'in' node=any any* > - """ - p2 = patcomp.compile_pattern(P2) - - def in_special_context(self, node, isiter): - if node.parent is None: - return False - results = {} - if (node.parent.parent is not None and - self.p1.match(node.parent.parent, results) and - results["node"] is node): - if isiter: - # iter(d.iterkeys()) -> iter(d.keys()), etc. - return results["func"].value in iter_exempt - else: - # list(d.keys()) -> list(d.keys()), etc. - return results["func"].value in fixer_util.consuming_calls - if not isiter: - return False - # for ... in d.iterkeys() -> for ... in d.keys(), etc. - return self.p2.match(node.parent, results) and results["node"] is node diff --git a/Lib/lib2to3/fixes/fix_except.py b/Lib/lib2to3/fixes/fix_except.py deleted file mode 100644 index 49bd3d5ab7d6cc..00000000000000 --- a/Lib/lib2to3/fixes/fix_except.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Fixer for except statements with named exceptions. - -The following cases will be converted: - -- "except E, T:" where T is a name: - - except E as T: - -- "except E, T:" where T is not a name, tuple or list: - - except E as t: - T = t - - This is done because the target of an "except" clause must be a - name. - -- "except E, T:" where T is a tuple or list literal: - - except E as t: - T = t.args -""" -# Author: Collin Winter - -# Local imports -from .. import pytree -from ..pgen2 import token -from .. import fixer_base -from ..fixer_util import Assign, Attr, Name, is_tuple, is_list, syms - -def find_excepts(nodes): - for i, n in enumerate(nodes): - if n.type == syms.except_clause: - if n.children[0].value == 'except': - yield (n, nodes[i+2]) - -class FixExcept(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - try_stmt< 'try' ':' (simple_stmt | suite) - cleanup=(except_clause ':' (simple_stmt | suite))+ - tail=(['except' ':' (simple_stmt | suite)] - ['else' ':' (simple_stmt | suite)] - ['finally' ':' (simple_stmt | suite)]) > - """ - - def transform(self, node, results): - syms = self.syms - - tail = [n.clone() for n in results["tail"]] - - try_cleanup = [ch.clone() for ch in results["cleanup"]] - for except_clause, e_suite in find_excepts(try_cleanup): - if len(except_clause.children) == 4: - (E, comma, N) = except_clause.children[1:4] - comma.replace(Name("as", prefix=" ")) - - if N.type != token.NAME: - # Generate a new N for the except clause - new_N = Name(self.new_name(), prefix=" ") - target = N.clone() - target.prefix = "" - N.replace(new_N) - new_N = new_N.clone() - - # Insert "old_N = new_N" as the first statement in - # the except body. This loop skips leading whitespace - # and indents - #TODO(cwinter) suite-cleanup - suite_stmts = e_suite.children - for i, stmt in enumerate(suite_stmts): - if isinstance(stmt, pytree.Node): - break - - # The assignment is different if old_N is a tuple or list - # In that case, the assignment is old_N = new_N.args - if is_tuple(N) or is_list(N): - assign = Assign(target, Attr(new_N, Name('args'))) - else: - assign = Assign(target, new_N) - - #TODO(cwinter) stopgap until children becomes a smart list - for child in reversed(suite_stmts[:i]): - e_suite.insert_child(0, child) - e_suite.insert_child(i, assign) - elif N.prefix == "": - # No space after a comma is legal; no space after "as", - # not so much. - N.prefix = " " - - #TODO(cwinter) fix this when children becomes a smart list - children = [c.clone() for c in node.children[:3]] + try_cleanup + tail - return pytree.Node(node.type, children) diff --git a/Lib/lib2to3/fixes/fix_exec.py b/Lib/lib2to3/fixes/fix_exec.py deleted file mode 100644 index ab921ee80cdf36..00000000000000 --- a/Lib/lib2to3/fixes/fix_exec.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for exec. - -This converts usages of the exec statement into calls to a built-in -exec() function. - -exec code in ns1, ns2 -> exec(code, ns1, ns2) -""" - -# Local imports -from .. import fixer_base -from ..fixer_util import Comma, Name, Call - - -class FixExec(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - exec_stmt< 'exec' a=any 'in' b=any [',' c=any] > - | - exec_stmt< 'exec' (not atom<'(' [any] ')'>) a=any > - """ - - def transform(self, node, results): - assert results - syms = self.syms - a = results["a"] - b = results.get("b") - c = results.get("c") - args = [a.clone()] - args[0].prefix = "" - if b is not None: - args.extend([Comma(), b.clone()]) - if c is not None: - args.extend([Comma(), c.clone()]) - - return Call(Name("exec"), args, prefix=node.prefix) diff --git a/Lib/lib2to3/fixes/fix_execfile.py b/Lib/lib2to3/fixes/fix_execfile.py deleted file mode 100644 index b6c786fd4e8b6a..00000000000000 --- a/Lib/lib2to3/fixes/fix_execfile.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for execfile. - -This converts usages of the execfile function into calls to the built-in -exec() function. -""" - -from .. import fixer_base -from ..fixer_util import (Comma, Name, Call, LParen, RParen, Dot, Node, - ArgList, String, syms) - - -class FixExecfile(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - power< 'execfile' trailer< '(' arglist< filename=any [',' globals=any [',' locals=any ] ] > ')' > > - | - power< 'execfile' trailer< '(' filename=any ')' > > - """ - - def transform(self, node, results): - assert results - filename = results["filename"] - globals = results.get("globals") - locals = results.get("locals") - - # Copy over the prefix from the right parentheses end of the execfile - # call. - execfile_paren = node.children[-1].children[-1].clone() - # Construct open().read(). - open_args = ArgList([filename.clone(), Comma(), String('"rb"', ' ')], - rparen=execfile_paren) - open_call = Node(syms.power, [Name("open"), open_args]) - read = [Node(syms.trailer, [Dot(), Name('read')]), - Node(syms.trailer, [LParen(), RParen()])] - open_expr = [open_call] + read - # Wrap the open call in a compile call. This is so the filename will be - # preserved in the execed code. - filename_arg = filename.clone() - filename_arg.prefix = " " - exec_str = String("'exec'", " ") - compile_args = open_expr + [Comma(), filename_arg, Comma(), exec_str] - compile_call = Call(Name("compile"), compile_args, "") - # Finally, replace the execfile call with an exec call. - args = [compile_call] - if globals is not None: - args.extend([Comma(), globals.clone()]) - if locals is not None: - args.extend([Comma(), locals.clone()]) - return Call(Name("exec"), args, prefix=node.prefix) diff --git a/Lib/lib2to3/fixes/fix_exitfunc.py b/Lib/lib2to3/fixes/fix_exitfunc.py deleted file mode 100644 index 2e47887afead36..00000000000000 --- a/Lib/lib2to3/fixes/fix_exitfunc.py +++ /dev/null @@ -1,72 +0,0 @@ -""" -Convert use of sys.exitfunc to use the atexit module. -""" - -# Author: Benjamin Peterson - -from lib2to3 import pytree, fixer_base -from lib2to3.fixer_util import Name, Attr, Call, Comma, Newline, syms - - -class FixExitfunc(fixer_base.BaseFix): - keep_line_order = True - BM_compatible = True - - PATTERN = """ - ( - sys_import=import_name<'import' - ('sys' - | - dotted_as_names< (any ',')* 'sys' (',' any)* > - ) - > - | - expr_stmt< - power< 'sys' trailer< '.' 'exitfunc' > > - '=' func=any > - ) - """ - - def __init__(self, *args): - super(FixExitfunc, self).__init__(*args) - - def start_tree(self, tree, filename): - super(FixExitfunc, self).start_tree(tree, filename) - self.sys_import = None - - def transform(self, node, results): - # First, find the sys import. We'll just hope it's global scope. - if "sys_import" in results: - if self.sys_import is None: - self.sys_import = results["sys_import"] - return - - func = results["func"].clone() - func.prefix = "" - register = pytree.Node(syms.power, - Attr(Name("atexit"), Name("register")) - ) - call = Call(register, [func], node.prefix) - node.replace(call) - - if self.sys_import is None: - # That's interesting. - self.warning(node, "Can't find sys import; Please add an atexit " - "import at the top of your file.") - return - - # Now add an atexit import after the sys import. - names = self.sys_import.children[1] - if names.type == syms.dotted_as_names: - names.append_child(Comma()) - names.append_child(Name("atexit", " ")) - else: - containing_stmt = self.sys_import.parent - position = containing_stmt.children.index(self.sys_import) - stmt_container = containing_stmt.parent - new_import = pytree.Node(syms.import_name, - [Name("import"), Name("atexit", " ")] - ) - new = pytree.Node(syms.simple_stmt, [new_import]) - containing_stmt.insert_child(position + 1, Newline()) - containing_stmt.insert_child(position + 2, new) diff --git a/Lib/lib2to3/fixes/fix_filter.py b/Lib/lib2to3/fixes/fix_filter.py deleted file mode 100644 index 38e9078f11ac88..00000000000000 --- a/Lib/lib2to3/fixes/fix_filter.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright 2007 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer that changes filter(F, X) into list(filter(F, X)). - -We avoid the transformation if the filter() call is directly contained -in iter(<>), list(<>), tuple(<>), sorted(<>), ...join(<>), or -for V in <>:. - -NOTE: This is still not correct if the original code was depending on -filter(F, X) to return a string if X is a string and a tuple if X is a -tuple. That would require type inference, which we don't do. Let -Python 2.6 figure it out. -""" - -# Local imports -from .. import fixer_base -from ..pytree import Node -from ..pygram import python_symbols as syms -from ..fixer_util import Name, ArgList, ListComp, in_special_context, parenthesize - - -class FixFilter(fixer_base.ConditionalFix): - BM_compatible = True - - PATTERN = """ - filter_lambda=power< - 'filter' - trailer< - '(' - arglist< - lambdef< 'lambda' - (fp=NAME | vfpdef< '(' fp=NAME ')'> ) ':' xp=any - > - ',' - it=any - > - ')' - > - [extra_trailers=trailer*] - > - | - power< - 'filter' - trailer< '(' arglist< none='None' ',' seq=any > ')' > - [extra_trailers=trailer*] - > - | - power< - 'filter' - args=trailer< '(' [any] ')' > - [extra_trailers=trailer*] - > - """ - - skip_on = "future_builtins.filter" - - def transform(self, node, results): - if self.should_skip(node): - return - - trailers = [] - if 'extra_trailers' in results: - for t in results['extra_trailers']: - trailers.append(t.clone()) - - if "filter_lambda" in results: - xp = results.get("xp").clone() - if xp.type == syms.test: - xp.prefix = "" - xp = parenthesize(xp) - - new = ListComp(results.get("fp").clone(), - results.get("fp").clone(), - results.get("it").clone(), xp) - new = Node(syms.power, [new] + trailers, prefix="") - - elif "none" in results: - new = ListComp(Name("_f"), - Name("_f"), - results["seq"].clone(), - Name("_f")) - new = Node(syms.power, [new] + trailers, prefix="") - - else: - if in_special_context(node): - return None - - args = results['args'].clone() - new = Node(syms.power, [Name("filter"), args], prefix="") - new = Node(syms.power, [Name("list"), ArgList([new])] + trailers) - new.prefix = "" - new.prefix = node.prefix - return new diff --git a/Lib/lib2to3/fixes/fix_funcattrs.py b/Lib/lib2to3/fixes/fix_funcattrs.py deleted file mode 100644 index 67f3e18e061bdb..00000000000000 --- a/Lib/lib2to3/fixes/fix_funcattrs.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Fix function attribute names (f.func_x -> f.__x__).""" -# Author: Collin Winter - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - - -class FixFuncattrs(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - power< any+ trailer< '.' attr=('func_closure' | 'func_doc' | 'func_globals' - | 'func_name' | 'func_defaults' | 'func_code' - | 'func_dict') > any* > - """ - - def transform(self, node, results): - attr = results["attr"][0] - attr.replace(Name(("__%s__" % attr.value[5:]), - prefix=attr.prefix)) diff --git a/Lib/lib2to3/fixes/fix_future.py b/Lib/lib2to3/fixes/fix_future.py deleted file mode 100644 index fbcb86af079133..00000000000000 --- a/Lib/lib2to3/fixes/fix_future.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Remove __future__ imports - -from __future__ import foo is replaced with an empty line. -""" -# Author: Christian Heimes - -# Local imports -from .. import fixer_base -from ..fixer_util import BlankLine - -class FixFuture(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """import_from< 'from' module_name="__future__" 'import' any >""" - - # This should be run last -- some things check for the import - run_order = 10 - - def transform(self, node, results): - new = BlankLine() - new.prefix = node.prefix - return new diff --git a/Lib/lib2to3/fixes/fix_getcwdu.py b/Lib/lib2to3/fixes/fix_getcwdu.py deleted file mode 100644 index 087eaedcb26f9c..00000000000000 --- a/Lib/lib2to3/fixes/fix_getcwdu.py +++ /dev/null @@ -1,19 +0,0 @@ -""" -Fixer that changes os.getcwdu() to os.getcwd(). -""" -# Author: Victor Stinner - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - -class FixGetcwdu(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - power< 'os' trailer< dot='.' name='getcwdu' > any* > - """ - - def transform(self, node, results): - name = results["name"] - name.replace(Name("getcwd", prefix=name.prefix)) diff --git a/Lib/lib2to3/fixes/fix_has_key.py b/Lib/lib2to3/fixes/fix_has_key.py deleted file mode 100644 index 439708c9923404..00000000000000 --- a/Lib/lib2to3/fixes/fix_has_key.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for has_key(). - -Calls to .has_key() methods are expressed in terms of the 'in' -operator: - - d.has_key(k) -> k in d - -CAVEATS: -1) While the primary target of this fixer is dict.has_key(), the - fixer will change any has_key() method call, regardless of its - class. - -2) Cases like this will not be converted: - - m = d.has_key - if m(k): - ... - - Only *calls* to has_key() are converted. While it is possible to - convert the above to something like - - m = d.__contains__ - if m(k): - ... - - this is currently not done. -""" - -# Local imports -from .. import pytree -from .. import fixer_base -from ..fixer_util import Name, parenthesize - - -class FixHasKey(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - anchor=power< - before=any+ - trailer< '.' 'has_key' > - trailer< - '(' - ( not(arglist | argument) arg=any ','> - ) - ')' - > - after=any* - > - | - negation=not_test< - 'not' - anchor=power< - before=any+ - trailer< '.' 'has_key' > - trailer< - '(' - ( not(arglist | argument) arg=any ','> - ) - ')' - > - > - > - """ - - def transform(self, node, results): - assert results - syms = self.syms - if (node.parent.type == syms.not_test and - self.pattern.match(node.parent)): - # Don't transform a node matching the first alternative of the - # pattern when its parent matches the second alternative - return None - negation = results.get("negation") - anchor = results["anchor"] - prefix = node.prefix - before = [n.clone() for n in results["before"]] - arg = results["arg"].clone() - after = results.get("after") - if after: - after = [n.clone() for n in after] - if arg.type in (syms.comparison, syms.not_test, syms.and_test, - syms.or_test, syms.test, syms.lambdef, syms.argument): - arg = parenthesize(arg) - if len(before) == 1: - before = before[0] - else: - before = pytree.Node(syms.power, before) - before.prefix = " " - n_op = Name("in", prefix=" ") - if negation: - n_not = Name("not", prefix=" ") - n_op = pytree.Node(syms.comp_op, (n_not, n_op)) - new = pytree.Node(syms.comparison, (arg, n_op, before)) - if after: - new = parenthesize(new) - new = pytree.Node(syms.power, (new,) + tuple(after)) - if node.parent.type in (syms.comparison, syms.expr, syms.xor_expr, - syms.and_expr, syms.shift_expr, - syms.arith_expr, syms.term, - syms.factor, syms.power): - new = parenthesize(new) - new.prefix = prefix - return new diff --git a/Lib/lib2to3/fixes/fix_idioms.py b/Lib/lib2to3/fixes/fix_idioms.py deleted file mode 100644 index 6905913d7cb79d..00000000000000 --- a/Lib/lib2to3/fixes/fix_idioms.py +++ /dev/null @@ -1,152 +0,0 @@ -"""Adjust some old Python 2 idioms to their modern counterparts. - -* Change some type comparisons to isinstance() calls: - type(x) == T -> isinstance(x, T) - type(x) is T -> isinstance(x, T) - type(x) != T -> not isinstance(x, T) - type(x) is not T -> not isinstance(x, T) - -* Change "while 1:" into "while True:". - -* Change both - - v = list(EXPR) - v.sort() - foo(v) - -and the more general - - v = EXPR - v.sort() - foo(v) - -into - - v = sorted(EXPR) - foo(v) -""" -# Author: Jacques Frechet, Collin Winter - -# Local imports -from .. import fixer_base -from ..fixer_util import Call, Comma, Name, Node, BlankLine, syms - -CMP = "(n='!=' | '==' | 'is' | n=comp_op< 'is' 'not' >)" -TYPE = "power< 'type' trailer< '(' x=any ')' > >" - -class FixIdioms(fixer_base.BaseFix): - explicit = True # The user must ask for this fixer - - PATTERN = r""" - isinstance=comparison< %s %s T=any > - | - isinstance=comparison< T=any %s %s > - | - while_stmt< 'while' while='1' ':' any+ > - | - sorted=any< - any* - simple_stmt< - expr_stmt< id1=any '=' - power< list='list' trailer< '(' (not arglist) any ')' > > - > - '\n' - > - sort= - simple_stmt< - power< id2=any - trailer< '.' 'sort' > trailer< '(' ')' > - > - '\n' - > - next=any* - > - | - sorted=any< - any* - simple_stmt< expr_stmt< id1=any '=' expr=any > '\n' > - sort= - simple_stmt< - power< id2=any - trailer< '.' 'sort' > trailer< '(' ')' > - > - '\n' - > - next=any* - > - """ % (TYPE, CMP, CMP, TYPE) - - def match(self, node): - r = super(FixIdioms, self).match(node) - # If we've matched one of the sort/sorted subpatterns above, we - # want to reject matches where the initial assignment and the - # subsequent .sort() call involve different identifiers. - if r and "sorted" in r: - if r["id1"] == r["id2"]: - return r - return None - return r - - def transform(self, node, results): - if "isinstance" in results: - return self.transform_isinstance(node, results) - elif "while" in results: - return self.transform_while(node, results) - elif "sorted" in results: - return self.transform_sort(node, results) - else: - raise RuntimeError("Invalid match") - - def transform_isinstance(self, node, results): - x = results["x"].clone() # The thing inside of type() - T = results["T"].clone() # The type being compared against - x.prefix = "" - T.prefix = " " - test = Call(Name("isinstance"), [x, Comma(), T]) - if "n" in results: - test.prefix = " " - test = Node(syms.not_test, [Name("not"), test]) - test.prefix = node.prefix - return test - - def transform_while(self, node, results): - one = results["while"] - one.replace(Name("True", prefix=one.prefix)) - - def transform_sort(self, node, results): - sort_stmt = results["sort"] - next_stmt = results["next"] - list_call = results.get("list") - simple_expr = results.get("expr") - - if list_call: - list_call.replace(Name("sorted", prefix=list_call.prefix)) - elif simple_expr: - new = simple_expr.clone() - new.prefix = "" - simple_expr.replace(Call(Name("sorted"), [new], - prefix=simple_expr.prefix)) - else: - raise RuntimeError("should not have reached here") - sort_stmt.remove() - - btwn = sort_stmt.prefix - # Keep any prefix lines between the sort_stmt and the list_call and - # shove them right after the sorted() call. - if "\n" in btwn: - if next_stmt: - # The new prefix should be everything from the sort_stmt's - # prefix up to the last newline, then the old prefix after a new - # line. - prefix_lines = (btwn.rpartition("\n")[0], next_stmt[0].prefix) - next_stmt[0].prefix = "\n".join(prefix_lines) - else: - assert list_call.parent - assert list_call.next_sibling is None - # Put a blank line after list_call and set its prefix. - end_line = BlankLine() - list_call.parent.append_child(end_line) - assert list_call.next_sibling is end_line - # The new prefix should be everything up to the first new line - # of sort_stmt's prefix. - end_line.prefix = btwn.rpartition("\n")[0] diff --git a/Lib/lib2to3/fixes/fix_import.py b/Lib/lib2to3/fixes/fix_import.py deleted file mode 100644 index 734ca294699c36..00000000000000 --- a/Lib/lib2to3/fixes/fix_import.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Fixer for import statements. -If spam is being imported from the local directory, this import: - from spam import eggs -Becomes: - from .spam import eggs - -And this import: - import spam -Becomes: - from . import spam -""" - -# Local imports -from .. import fixer_base -from os.path import dirname, join, exists, sep -from ..fixer_util import FromImport, syms, token - - -def traverse_imports(names): - """ - Walks over all the names imported in a dotted_as_names node. - """ - pending = [names] - while pending: - node = pending.pop() - if node.type == token.NAME: - yield node.value - elif node.type == syms.dotted_name: - yield "".join([ch.value for ch in node.children]) - elif node.type == syms.dotted_as_name: - pending.append(node.children[0]) - elif node.type == syms.dotted_as_names: - pending.extend(node.children[::-2]) - else: - raise AssertionError("unknown node type") - - -class FixImport(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - import_from< 'from' imp=any 'import' ['('] any [')'] > - | - import_name< 'import' imp=any > - """ - - def start_tree(self, tree, name): - super(FixImport, self).start_tree(tree, name) - self.skip = "absolute_import" in tree.future_features - - def transform(self, node, results): - if self.skip: - return - imp = results['imp'] - - if node.type == syms.import_from: - # Some imps are top-level (eg: 'import ham') - # some are first level (eg: 'import ham.eggs') - # some are third level (eg: 'import ham.eggs as spam') - # Hence, the loop - while not hasattr(imp, 'value'): - imp = imp.children[0] - if self.probably_a_local_import(imp.value): - imp.value = "." + imp.value - imp.changed() - else: - have_local = False - have_absolute = False - for mod_name in traverse_imports(imp): - if self.probably_a_local_import(mod_name): - have_local = True - else: - have_absolute = True - if have_absolute: - if have_local: - # We won't handle both sibling and absolute imports in the - # same statement at the moment. - self.warning(node, "absolute and local imports together") - return - - new = FromImport(".", [imp]) - new.prefix = node.prefix - return new - - def probably_a_local_import(self, imp_name): - if imp_name.startswith("."): - # Relative imports are certainly not local imports. - return False - imp_name = imp_name.split(".", 1)[0] - base_path = dirname(self.filename) - base_path = join(base_path, imp_name) - # If there is no __init__.py next to the file its not in a package - # so can't be a relative import. - if not exists(join(dirname(base_path), "__init__.py")): - return False - for ext in [".py", sep, ".pyc", ".so", ".sl", ".pyd"]: - if exists(base_path + ext): - return True - return False diff --git a/Lib/lib2to3/fixes/fix_imports.py b/Lib/lib2to3/fixes/fix_imports.py deleted file mode 100644 index aaf4f2f642efb5..00000000000000 --- a/Lib/lib2to3/fixes/fix_imports.py +++ /dev/null @@ -1,145 +0,0 @@ -"""Fix incompatible imports and module references.""" -# Authors: Collin Winter, Nick Edds - -# Local imports -from .. import fixer_base -from ..fixer_util import Name, attr_chain - -MAPPING = {'StringIO': 'io', - 'cStringIO': 'io', - 'cPickle': 'pickle', - '__builtin__' : 'builtins', - 'copy_reg': 'copyreg', - 'Queue': 'queue', - 'SocketServer': 'socketserver', - 'ConfigParser': 'configparser', - 'repr': 'reprlib', - 'FileDialog': 'tkinter.filedialog', - 'tkFileDialog': 'tkinter.filedialog', - 'SimpleDialog': 'tkinter.simpledialog', - 'tkSimpleDialog': 'tkinter.simpledialog', - 'tkColorChooser': 'tkinter.colorchooser', - 'tkCommonDialog': 'tkinter.commondialog', - 'Dialog': 'tkinter.dialog', - 'Tkdnd': 'tkinter.dnd', - 'tkFont': 'tkinter.font', - 'tkMessageBox': 'tkinter.messagebox', - 'ScrolledText': 'tkinter.scrolledtext', - 'Tkconstants': 'tkinter.constants', - 'Tix': 'tkinter.tix', - 'ttk': 'tkinter.ttk', - 'Tkinter': 'tkinter', - 'markupbase': '_markupbase', - '_winreg': 'winreg', - 'thread': '_thread', - 'dummy_thread': '_dummy_thread', - # anydbm and whichdb are handled by fix_imports2 - 'dbhash': 'dbm.bsd', - 'dumbdbm': 'dbm.dumb', - 'dbm': 'dbm.ndbm', - 'gdbm': 'dbm.gnu', - 'xmlrpclib': 'xmlrpc.client', - 'DocXMLRPCServer': 'xmlrpc.server', - 'SimpleXMLRPCServer': 'xmlrpc.server', - 'httplib': 'http.client', - 'htmlentitydefs' : 'html.entities', - 'HTMLParser' : 'html.parser', - 'Cookie': 'http.cookies', - 'cookielib': 'http.cookiejar', - 'BaseHTTPServer': 'http.server', - 'SimpleHTTPServer': 'http.server', - 'CGIHTTPServer': 'http.server', - #'test.test_support': 'test.support', - 'commands': 'subprocess', - 'UserString' : 'collections', - 'UserList' : 'collections', - 'urlparse' : 'urllib.parse', - 'robotparser' : 'urllib.robotparser', -} - - -def alternates(members): - return "(" + "|".join(map(repr, members)) + ")" - - -def build_pattern(mapping=MAPPING): - mod_list = ' | '.join(["module_name='%s'" % key for key in mapping]) - bare_names = alternates(mapping.keys()) - - yield """name_import=import_name< 'import' ((%s) | - multiple_imports=dotted_as_names< any* (%s) any* >) > - """ % (mod_list, mod_list) - yield """import_from< 'from' (%s) 'import' ['('] - ( any | import_as_name< any 'as' any > | - import_as_names< any* >) [')'] > - """ % mod_list - yield """import_name< 'import' (dotted_as_name< (%s) 'as' any > | - multiple_imports=dotted_as_names< - any* dotted_as_name< (%s) 'as' any > any* >) > - """ % (mod_list, mod_list) - - # Find usages of module members in code e.g. thread.foo(bar) - yield "power< bare_with_attr=(%s) trailer<'.' any > any* >" % bare_names - - -class FixImports(fixer_base.BaseFix): - - BM_compatible = True - keep_line_order = True - # This is overridden in fix_imports2. - mapping = MAPPING - - # We want to run this fixer late, so fix_import doesn't try to make stdlib - # renames into relative imports. - run_order = 6 - - def build_pattern(self): - return "|".join(build_pattern(self.mapping)) - - def compile_pattern(self): - # We override this, so MAPPING can be pragmatically altered and the - # changes will be reflected in PATTERN. - self.PATTERN = self.build_pattern() - super(FixImports, self).compile_pattern() - - # Don't match the node if it's within another match. - def match(self, node): - match = super(FixImports, self).match - results = match(node) - if results: - # Module usage could be in the trailer of an attribute lookup, so we - # might have nested matches when "bare_with_attr" is present. - if "bare_with_attr" not in results and \ - any(match(obj) for obj in attr_chain(node, "parent")): - return False - return results - return False - - def start_tree(self, tree, filename): - super(FixImports, self).start_tree(tree, filename) - self.replace = {} - - def transform(self, node, results): - import_mod = results.get("module_name") - if import_mod: - mod_name = import_mod.value - new_name = self.mapping[mod_name] - import_mod.replace(Name(new_name, prefix=import_mod.prefix)) - if "name_import" in results: - # If it's not a "from x import x, y" or "import x as y" import, - # marked its usage to be replaced. - self.replace[mod_name] = new_name - if "multiple_imports" in results: - # This is a nasty hack to fix multiple imports on a line (e.g., - # "import StringIO, urlparse"). The problem is that I can't - # figure out an easy way to make a pattern recognize the keys of - # MAPPING randomly sprinkled in an import statement. - results = self.match(node) - if results: - self.transform(node, results) - else: - # Replace usage of the module. - bare_name = results["bare_with_attr"][0] - new_name = self.replace.get(bare_name.value) - if new_name: - bare_name.replace(Name(new_name, prefix=bare_name.prefix)) diff --git a/Lib/lib2to3/fixes/fix_imports2.py b/Lib/lib2to3/fixes/fix_imports2.py deleted file mode 100644 index 9a33c67b1dc194..00000000000000 --- a/Lib/lib2to3/fixes/fix_imports2.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Fix incompatible imports and module references that must be fixed after -fix_imports.""" -from . import fix_imports - - -MAPPING = { - 'whichdb': 'dbm', - 'anydbm': 'dbm', - } - - -class FixImports2(fix_imports.FixImports): - - run_order = 7 - - mapping = MAPPING diff --git a/Lib/lib2to3/fixes/fix_input.py b/Lib/lib2to3/fixes/fix_input.py deleted file mode 100644 index 9cf9a48c471f35..00000000000000 --- a/Lib/lib2to3/fixes/fix_input.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Fixer that changes input(...) into eval(input(...)).""" -# Author: Andre Roberge - -# Local imports -from .. import fixer_base -from ..fixer_util import Call, Name -from .. import patcomp - - -context = patcomp.compile_pattern("power< 'eval' trailer< '(' any ')' > >") - - -class FixInput(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - power< 'input' args=trailer< '(' [any] ')' > > - """ - - def transform(self, node, results): - # If we're already wrapped in an eval() call, we're done. - if context.match(node.parent.parent): - return - - new = node.clone() - new.prefix = "" - return Call(Name("eval"), [new], prefix=node.prefix) diff --git a/Lib/lib2to3/fixes/fix_intern.py b/Lib/lib2to3/fixes/fix_intern.py deleted file mode 100644 index d752843092aacd..00000000000000 --- a/Lib/lib2to3/fixes/fix_intern.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2006 Georg Brandl. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for intern(). - -intern(s) -> sys.intern(s)""" - -# Local imports -from .. import fixer_base -from ..fixer_util import ImportAndCall, touch_import - - -class FixIntern(fixer_base.BaseFix): - BM_compatible = True - order = "pre" - - PATTERN = """ - power< 'intern' - trailer< lpar='(' - ( not(arglist | argument) any ','> ) - rpar=')' > - after=any* - > - """ - - def transform(self, node, results): - if results: - # I feel like we should be able to express this logic in the - # PATTERN above but I don't know how to do it so... - obj = results['obj'] - if obj: - if (obj.type == self.syms.argument and - obj.children[0].value in {'**', '*'}): - return # Make no change. - names = ('sys', 'intern') - new = ImportAndCall(node, results, names) - touch_import(None, 'sys', node) - return new diff --git a/Lib/lib2to3/fixes/fix_isinstance.py b/Lib/lib2to3/fixes/fix_isinstance.py deleted file mode 100644 index bebb1de120424b..00000000000000 --- a/Lib/lib2to3/fixes/fix_isinstance.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2008 Armin Ronacher. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer that cleans up a tuple argument to isinstance after the tokens -in it were fixed. This is mainly used to remove double occurrences of -tokens as a leftover of the long -> int / unicode -> str conversion. - -eg. isinstance(x, (int, long)) -> isinstance(x, (int, int)) - -> isinstance(x, int) -""" - -from .. import fixer_base -from ..fixer_util import token - - -class FixIsinstance(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - power< - 'isinstance' - trailer< '(' arglist< any ',' atom< '(' - args=testlist_gexp< any+ > - ')' > > ')' > - > - """ - - run_order = 6 - - def transform(self, node, results): - names_inserted = set() - testlist = results["args"] - args = testlist.children - new_args = [] - iterator = enumerate(args) - for idx, arg in iterator: - if arg.type == token.NAME and arg.value in names_inserted: - if idx < len(args) - 1 and args[idx + 1].type == token.COMMA: - next(iterator) - continue - else: - new_args.append(arg) - if arg.type == token.NAME: - names_inserted.add(arg.value) - if new_args and new_args[-1].type == token.COMMA: - del new_args[-1] - if len(new_args) == 1: - atom = testlist.parent - new_args[0].prefix = atom.prefix - atom.replace(new_args[0]) - else: - args[:] = new_args - node.changed() diff --git a/Lib/lib2to3/fixes/fix_itertools.py b/Lib/lib2to3/fixes/fix_itertools.py deleted file mode 100644 index 8e78d6c689f439..00000000000000 --- a/Lib/lib2to3/fixes/fix_itertools.py +++ /dev/null @@ -1,43 +0,0 @@ -""" Fixer for itertools.(imap|ifilter|izip) --> (map|filter|zip) and - itertools.ifilterfalse --> itertools.filterfalse (bugs 2360-2363) - - imports from itertools are fixed in fix_itertools_import.py - - If itertools is imported as something else (ie: import itertools as it; - it.izip(spam, eggs)) method calls will not get fixed. - """ - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - -class FixItertools(fixer_base.BaseFix): - BM_compatible = True - it_funcs = "('imap'|'ifilter'|'izip'|'izip_longest'|'ifilterfalse')" - PATTERN = """ - power< it='itertools' - trailer< - dot='.' func=%(it_funcs)s > trailer< '(' [any] ')' > > - | - power< func=%(it_funcs)s trailer< '(' [any] ')' > > - """ %(locals()) - - # Needs to be run after fix_(map|zip|filter) - run_order = 6 - - def transform(self, node, results): - prefix = None - func = results['func'][0] - if ('it' in results and - func.value not in ('ifilterfalse', 'izip_longest')): - dot, it = (results['dot'], results['it']) - # Remove the 'itertools' - prefix = it.prefix - it.remove() - # Replace the node which contains ('.', 'function') with the - # function (to be consistent with the second part of the pattern) - dot.remove() - func.parent.replace(func) - - prefix = prefix or func.prefix - func.replace(Name(func.value[1:], prefix=prefix)) diff --git a/Lib/lib2to3/fixes/fix_itertools_imports.py b/Lib/lib2to3/fixes/fix_itertools_imports.py deleted file mode 100644 index 0ddbc7b8422991..00000000000000 --- a/Lib/lib2to3/fixes/fix_itertools_imports.py +++ /dev/null @@ -1,57 +0,0 @@ -""" Fixer for imports of itertools.(imap|ifilter|izip|ifilterfalse) """ - -# Local imports -from lib2to3 import fixer_base -from lib2to3.fixer_util import BlankLine, syms, token - - -class FixItertoolsImports(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - import_from< 'from' 'itertools' 'import' imports=any > - """ %(locals()) - - def transform(self, node, results): - imports = results['imports'] - if imports.type == syms.import_as_name or not imports.children: - children = [imports] - else: - children = imports.children - for child in children[::2]: - if child.type == token.NAME: - member = child.value - name_node = child - elif child.type == token.STAR: - # Just leave the import as is. - return - else: - assert child.type == syms.import_as_name - name_node = child.children[0] - member_name = name_node.value - if member_name in ('imap', 'izip', 'ifilter'): - child.value = None - child.remove() - elif member_name in ('ifilterfalse', 'izip_longest'): - node.changed() - name_node.value = ('filterfalse' if member_name[1] == 'f' - else 'zip_longest') - - # Make sure the import statement is still sane - children = imports.children[:] or [imports] - remove_comma = True - for child in children: - if remove_comma and child.type == token.COMMA: - child.remove() - else: - remove_comma ^= True - - while children and children[-1].type == token.COMMA: - children.pop().remove() - - # If there are no imports left, just get rid of the entire statement - if (not (imports.children or getattr(imports, 'value', None)) or - imports.parent is None): - p = node.prefix - node = BlankLine() - node.prefix = p - return node diff --git a/Lib/lib2to3/fixes/fix_long.py b/Lib/lib2to3/fixes/fix_long.py deleted file mode 100644 index f227c9f4981538..00000000000000 --- a/Lib/lib2to3/fixes/fix_long.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer that turns 'long' into 'int' everywhere. -""" - -# Local imports -from lib2to3 import fixer_base -from lib2to3.fixer_util import is_probably_builtin - - -class FixLong(fixer_base.BaseFix): - BM_compatible = True - PATTERN = "'long'" - - def transform(self, node, results): - if is_probably_builtin(node): - node.value = "int" - node.changed() diff --git a/Lib/lib2to3/fixes/fix_map.py b/Lib/lib2to3/fixes/fix_map.py deleted file mode 100644 index 78cf81c6f94098..00000000000000 --- a/Lib/lib2to3/fixes/fix_map.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright 2007 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer that changes map(F, ...) into list(map(F, ...)) unless there -exists a 'from future_builtins import map' statement in the top-level -namespace. - -As a special case, map(None, X) is changed into list(X). (This is -necessary because the semantics are changed in this case -- the new -map(None, X) is equivalent to [(x,) for x in X].) - -We avoid the transformation (except for the special case mentioned -above) if the map() call is directly contained in iter(<>), list(<>), -tuple(<>), sorted(<>), ...join(<>), or for V in <>:. - -NOTE: This is still not correct if the original code was depending on -map(F, X, Y, ...) to go on until the longest argument is exhausted, -substituting None for missing values -- like zip(), it now stops as -soon as the shortest argument is exhausted. -""" - -# Local imports -from ..pgen2 import token -from .. import fixer_base -from ..fixer_util import Name, ArgList, Call, ListComp, in_special_context -from ..pygram import python_symbols as syms -from ..pytree import Node - - -class FixMap(fixer_base.ConditionalFix): - BM_compatible = True - - PATTERN = """ - map_none=power< - 'map' - trailer< '(' arglist< 'None' ',' arg=any [','] > ')' > - [extra_trailers=trailer*] - > - | - map_lambda=power< - 'map' - trailer< - '(' - arglist< - lambdef< 'lambda' - (fp=NAME | vfpdef< '(' fp=NAME ')'> ) ':' xp=any - > - ',' - it=any - > - ')' - > - [extra_trailers=trailer*] - > - | - power< - 'map' args=trailer< '(' [any] ')' > - [extra_trailers=trailer*] - > - """ - - skip_on = 'future_builtins.map' - - def transform(self, node, results): - if self.should_skip(node): - return - - trailers = [] - if 'extra_trailers' in results: - for t in results['extra_trailers']: - trailers.append(t.clone()) - - if node.parent.type == syms.simple_stmt: - self.warning(node, "You should use a for loop here") - new = node.clone() - new.prefix = "" - new = Call(Name("list"), [new]) - elif "map_lambda" in results: - new = ListComp(results["xp"].clone(), - results["fp"].clone(), - results["it"].clone()) - new = Node(syms.power, [new] + trailers, prefix="") - - else: - if "map_none" in results: - new = results["arg"].clone() - new.prefix = "" - else: - if "args" in results: - args = results["args"] - if args.type == syms.trailer and \ - args.children[1].type == syms.arglist and \ - args.children[1].children[0].type == token.NAME and \ - args.children[1].children[0].value == "None": - self.warning(node, "cannot convert map(None, ...) " - "with multiple arguments because map() " - "now truncates to the shortest sequence") - return - - new = Node(syms.power, [Name("map"), args.clone()]) - new.prefix = "" - - if in_special_context(node): - return None - - new = Node(syms.power, [Name("list"), ArgList([new])] + trailers) - new.prefix = "" - - new.prefix = node.prefix - return new diff --git a/Lib/lib2to3/fixes/fix_metaclass.py b/Lib/lib2to3/fixes/fix_metaclass.py deleted file mode 100644 index fe547b2228072a..00000000000000 --- a/Lib/lib2to3/fixes/fix_metaclass.py +++ /dev/null @@ -1,228 +0,0 @@ -"""Fixer for __metaclass__ = X -> (metaclass=X) methods. - - The various forms of classef (inherits nothing, inherits once, inherits - many) don't parse the same in the CST so we look at ALL classes for - a __metaclass__ and if we find one normalize the inherits to all be - an arglist. - - For one-liner classes ('class X: pass') there is no indent/dedent so - we normalize those into having a suite. - - Moving the __metaclass__ into the classdef can also cause the class - body to be empty so there is some special casing for that as well. - - This fixer also tries very hard to keep original indenting and spacing - in all those corner cases. - -""" -# Author: Jack Diederich - -# Local imports -from .. import fixer_base -from ..pygram import token -from ..fixer_util import syms, Node, Leaf - - -def has_metaclass(parent): - """ we have to check the cls_node without changing it. - There are two possibilities: - 1) clsdef => suite => simple_stmt => expr_stmt => Leaf('__meta') - 2) clsdef => simple_stmt => expr_stmt => Leaf('__meta') - """ - for node in parent.children: - if node.type == syms.suite: - return has_metaclass(node) - elif node.type == syms.simple_stmt and node.children: - expr_node = node.children[0] - if expr_node.type == syms.expr_stmt and expr_node.children: - left_side = expr_node.children[0] - if isinstance(left_side, Leaf) and \ - left_side.value == '__metaclass__': - return True - return False - - -def fixup_parse_tree(cls_node): - """ one-line classes don't get a suite in the parse tree so we add - one to normalize the tree - """ - for node in cls_node.children: - if node.type == syms.suite: - # already in the preferred format, do nothing - return - - # !%@#! one-liners have no suite node, we have to fake one up - for i, node in enumerate(cls_node.children): - if node.type == token.COLON: - break - else: - raise ValueError("No class suite and no ':'!") - - # move everything into a suite node - suite = Node(syms.suite, []) - while cls_node.children[i+1:]: - move_node = cls_node.children[i+1] - suite.append_child(move_node.clone()) - move_node.remove() - cls_node.append_child(suite) - node = suite - - -def fixup_simple_stmt(parent, i, stmt_node): - """ if there is a semi-colon all the parts count as part of the same - simple_stmt. We just want the __metaclass__ part so we move - everything after the semi-colon into its own simple_stmt node - """ - for semi_ind, node in enumerate(stmt_node.children): - if node.type == token.SEMI: # *sigh* - break - else: - return - - node.remove() # kill the semicolon - new_expr = Node(syms.expr_stmt, []) - new_stmt = Node(syms.simple_stmt, [new_expr]) - while stmt_node.children[semi_ind:]: - move_node = stmt_node.children[semi_ind] - new_expr.append_child(move_node.clone()) - move_node.remove() - parent.insert_child(i, new_stmt) - new_leaf1 = new_stmt.children[0].children[0] - old_leaf1 = stmt_node.children[0].children[0] - new_leaf1.prefix = old_leaf1.prefix - - -def remove_trailing_newline(node): - if node.children and node.children[-1].type == token.NEWLINE: - node.children[-1].remove() - - -def find_metas(cls_node): - # find the suite node (Mmm, sweet nodes) - for node in cls_node.children: - if node.type == syms.suite: - break - else: - raise ValueError("No class suite!") - - # look for simple_stmt[ expr_stmt[ Leaf('__metaclass__') ] ] - for i, simple_node in list(enumerate(node.children)): - if simple_node.type == syms.simple_stmt and simple_node.children: - expr_node = simple_node.children[0] - if expr_node.type == syms.expr_stmt and expr_node.children: - # Check if the expr_node is a simple assignment. - left_node = expr_node.children[0] - if isinstance(left_node, Leaf) and \ - left_node.value == '__metaclass__': - # We found an assignment to __metaclass__. - fixup_simple_stmt(node, i, simple_node) - remove_trailing_newline(simple_node) - yield (node, i, simple_node) - - -def fixup_indent(suite): - """ If an INDENT is followed by a thing with a prefix then nuke the prefix - Otherwise we get in trouble when removing __metaclass__ at suite start - """ - kids = suite.children[::-1] - # find the first indent - while kids: - node = kids.pop() - if node.type == token.INDENT: - break - - # find the first Leaf - while kids: - node = kids.pop() - if isinstance(node, Leaf) and node.type != token.DEDENT: - if node.prefix: - node.prefix = '' - return - else: - kids.extend(node.children[::-1]) - - -class FixMetaclass(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - classdef - """ - - def transform(self, node, results): - if not has_metaclass(node): - return - - fixup_parse_tree(node) - - # find metaclasses, keep the last one - last_metaclass = None - for suite, i, stmt in find_metas(node): - last_metaclass = stmt - stmt.remove() - - text_type = node.children[0].type # always Leaf(nnn, 'class') - - # figure out what kind of classdef we have - if len(node.children) == 7: - # Node(classdef, ['class', 'name', '(', arglist, ')', ':', suite]) - # 0 1 2 3 4 5 6 - if node.children[3].type == syms.arglist: - arglist = node.children[3] - # Node(classdef, ['class', 'name', '(', 'Parent', ')', ':', suite]) - else: - parent = node.children[3].clone() - arglist = Node(syms.arglist, [parent]) - node.set_child(3, arglist) - elif len(node.children) == 6: - # Node(classdef, ['class', 'name', '(', ')', ':', suite]) - # 0 1 2 3 4 5 - arglist = Node(syms.arglist, []) - node.insert_child(3, arglist) - elif len(node.children) == 4: - # Node(classdef, ['class', 'name', ':', suite]) - # 0 1 2 3 - arglist = Node(syms.arglist, []) - node.insert_child(2, Leaf(token.RPAR, ')')) - node.insert_child(2, arglist) - node.insert_child(2, Leaf(token.LPAR, '(')) - else: - raise ValueError("Unexpected class definition") - - # now stick the metaclass in the arglist - meta_txt = last_metaclass.children[0].children[0] - meta_txt.value = 'metaclass' - orig_meta_prefix = meta_txt.prefix - - if arglist.children: - arglist.append_child(Leaf(token.COMMA, ',')) - meta_txt.prefix = ' ' - else: - meta_txt.prefix = '' - - # compact the expression "metaclass = Meta" -> "metaclass=Meta" - expr_stmt = last_metaclass.children[0] - assert expr_stmt.type == syms.expr_stmt - expr_stmt.children[1].prefix = '' - expr_stmt.children[2].prefix = '' - - arglist.append_child(last_metaclass) - - fixup_indent(suite) - - # check for empty suite - if not suite.children: - # one-liner that was just __metaclass_ - suite.remove() - pass_leaf = Leaf(text_type, 'pass') - pass_leaf.prefix = orig_meta_prefix - node.append_child(pass_leaf) - node.append_child(Leaf(token.NEWLINE, '\n')) - - elif len(suite.children) > 1 and \ - (suite.children[-2].type == token.INDENT and - suite.children[-1].type == token.DEDENT): - # there was only one line in the class body and it was __metaclass__ - pass_leaf = Leaf(text_type, 'pass') - suite.insert_child(-1, pass_leaf) - suite.insert_child(-1, Leaf(token.NEWLINE, '\n')) diff --git a/Lib/lib2to3/fixes/fix_methodattrs.py b/Lib/lib2to3/fixes/fix_methodattrs.py deleted file mode 100644 index 7f9004f00e6e8f..00000000000000 --- a/Lib/lib2to3/fixes/fix_methodattrs.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Fix bound method attributes (method.im_? -> method.__?__). -""" -# Author: Christian Heimes - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - -MAP = { - "im_func" : "__func__", - "im_self" : "__self__", - "im_class" : "__self__.__class__" - } - -class FixMethodattrs(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - power< any+ trailer< '.' attr=('im_func' | 'im_self' | 'im_class') > any* > - """ - - def transform(self, node, results): - attr = results["attr"][0] - new = MAP[attr.value] - attr.replace(Name(new, prefix=attr.prefix)) diff --git a/Lib/lib2to3/fixes/fix_ne.py b/Lib/lib2to3/fixes/fix_ne.py deleted file mode 100644 index e3ee10f4a63e0c..00000000000000 --- a/Lib/lib2to3/fixes/fix_ne.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer that turns <> into !=.""" - -# Local imports -from .. import pytree -from ..pgen2 import token -from .. import fixer_base - - -class FixNe(fixer_base.BaseFix): - # This is so simple that we don't need the pattern compiler. - - _accept_type = token.NOTEQUAL - - def match(self, node): - # Override - return node.value == "<>" - - def transform(self, node, results): - new = pytree.Leaf(token.NOTEQUAL, "!=", prefix=node.prefix) - return new diff --git a/Lib/lib2to3/fixes/fix_next.py b/Lib/lib2to3/fixes/fix_next.py deleted file mode 100644 index 9f6305e1d49dc5..00000000000000 --- a/Lib/lib2to3/fixes/fix_next.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Fixer for it.next() -> next(it), per PEP 3114.""" -# Author: Collin Winter - -# Things that currently aren't covered: -# - listcomp "next" names aren't warned -# - "with" statement targets aren't checked - -# Local imports -from ..pgen2 import token -from ..pygram import python_symbols as syms -from .. import fixer_base -from ..fixer_util import Name, Call, find_binding - -bind_warning = "Calls to builtin next() possibly shadowed by global binding" - - -class FixNext(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - power< base=any+ trailer< '.' attr='next' > trailer< '(' ')' > > - | - power< head=any+ trailer< '.' attr='next' > not trailer< '(' ')' > > - | - classdef< 'class' any+ ':' - suite< any* - funcdef< 'def' - name='next' - parameters< '(' NAME ')' > any+ > - any* > > - | - global=global_stmt< 'global' any* 'next' any* > - """ - - order = "pre" # Pre-order tree traversal - - def start_tree(self, tree, filename): - super(FixNext, self).start_tree(tree, filename) - - n = find_binding('next', tree) - if n: - self.warning(n, bind_warning) - self.shadowed_next = True - else: - self.shadowed_next = False - - def transform(self, node, results): - assert results - - base = results.get("base") - attr = results.get("attr") - name = results.get("name") - - if base: - if self.shadowed_next: - attr.replace(Name("__next__", prefix=attr.prefix)) - else: - base = [n.clone() for n in base] - base[0].prefix = "" - node.replace(Call(Name("next", prefix=node.prefix), base)) - elif name: - n = Name("__next__", prefix=name.prefix) - name.replace(n) - elif attr: - # We don't do this transformation if we're assigning to "x.next". - # Unfortunately, it doesn't seem possible to do this in PATTERN, - # so it's being done here. - if is_assign_target(node): - head = results["head"] - if "".join([str(n) for n in head]).strip() == '__builtin__': - self.warning(node, bind_warning) - return - attr.replace(Name("__next__")) - elif "global" in results: - self.warning(node, bind_warning) - self.shadowed_next = True - - -### The following functions help test if node is part of an assignment -### target. - -def is_assign_target(node): - assign = find_assign(node) - if assign is None: - return False - - for child in assign.children: - if child.type == token.EQUAL: - return False - elif is_subtree(child, node): - return True - return False - -def find_assign(node): - if node.type == syms.expr_stmt: - return node - if node.type == syms.simple_stmt or node.parent is None: - return None - return find_assign(node.parent) - -def is_subtree(root, node): - if root == node: - return True - return any(is_subtree(c, node) for c in root.children) diff --git a/Lib/lib2to3/fixes/fix_nonzero.py b/Lib/lib2to3/fixes/fix_nonzero.py deleted file mode 100644 index c2295969a7728f..00000000000000 --- a/Lib/lib2to3/fixes/fix_nonzero.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Fixer for __nonzero__ -> __bool__ methods.""" -# Author: Collin Winter - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - -class FixNonzero(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - classdef< 'class' any+ ':' - suite< any* - funcdef< 'def' name='__nonzero__' - parameters< '(' NAME ')' > any+ > - any* > > - """ - - def transform(self, node, results): - name = results["name"] - new = Name("__bool__", prefix=name.prefix) - name.replace(new) diff --git a/Lib/lib2to3/fixes/fix_numliterals.py b/Lib/lib2to3/fixes/fix_numliterals.py deleted file mode 100644 index 79207d4aa368ae..00000000000000 --- a/Lib/lib2to3/fixes/fix_numliterals.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Fixer that turns 1L into 1, 0755 into 0o755. -""" -# Copyright 2007 Georg Brandl. -# Licensed to PSF under a Contributor Agreement. - -# Local imports -from ..pgen2 import token -from .. import fixer_base -from ..fixer_util import Number - - -class FixNumliterals(fixer_base.BaseFix): - # This is so simple that we don't need the pattern compiler. - - _accept_type = token.NUMBER - - def match(self, node): - # Override - return (node.value.startswith("0") or node.value[-1] in "Ll") - - def transform(self, node, results): - val = node.value - if val[-1] in 'Ll': - val = val[:-1] - elif val.startswith('0') and val.isdigit() and len(set(val)) > 1: - val = "0o" + val[1:] - - return Number(val, prefix=node.prefix) diff --git a/Lib/lib2to3/fixes/fix_operator.py b/Lib/lib2to3/fixes/fix_operator.py deleted file mode 100644 index d303cd2018befb..00000000000000 --- a/Lib/lib2to3/fixes/fix_operator.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Fixer for operator functions. - -operator.isCallable(obj) -> callable(obj) -operator.sequenceIncludes(obj) -> operator.contains(obj) -operator.isSequenceType(obj) -> isinstance(obj, collections.abc.Sequence) -operator.isMappingType(obj) -> isinstance(obj, collections.abc.Mapping) -operator.isNumberType(obj) -> isinstance(obj, numbers.Number) -operator.repeat(obj, n) -> operator.mul(obj, n) -operator.irepeat(obj, n) -> operator.imul(obj, n) -""" - -import collections.abc - -# Local imports -from lib2to3 import fixer_base -from lib2to3.fixer_util import Call, Name, String, touch_import - - -def invocation(s): - def dec(f): - f.invocation = s - return f - return dec - - -class FixOperator(fixer_base.BaseFix): - BM_compatible = True - order = "pre" - - methods = """ - method=('isCallable'|'sequenceIncludes' - |'isSequenceType'|'isMappingType'|'isNumberType' - |'repeat'|'irepeat') - """ - obj = "'(' obj=any ')'" - PATTERN = """ - power< module='operator' - trailer< '.' %(methods)s > trailer< %(obj)s > > - | - power< %(methods)s trailer< %(obj)s > > - """ % dict(methods=methods, obj=obj) - - def transform(self, node, results): - method = self._check_method(node, results) - if method is not None: - return method(node, results) - - @invocation("operator.contains(%s)") - def _sequenceIncludes(self, node, results): - return self._handle_rename(node, results, "contains") - - @invocation("callable(%s)") - def _isCallable(self, node, results): - obj = results["obj"] - return Call(Name("callable"), [obj.clone()], prefix=node.prefix) - - @invocation("operator.mul(%s)") - def _repeat(self, node, results): - return self._handle_rename(node, results, "mul") - - @invocation("operator.imul(%s)") - def _irepeat(self, node, results): - return self._handle_rename(node, results, "imul") - - @invocation("isinstance(%s, collections.abc.Sequence)") - def _isSequenceType(self, node, results): - return self._handle_type2abc(node, results, "collections.abc", "Sequence") - - @invocation("isinstance(%s, collections.abc.Mapping)") - def _isMappingType(self, node, results): - return self._handle_type2abc(node, results, "collections.abc", "Mapping") - - @invocation("isinstance(%s, numbers.Number)") - def _isNumberType(self, node, results): - return self._handle_type2abc(node, results, "numbers", "Number") - - def _handle_rename(self, node, results, name): - method = results["method"][0] - method.value = name - method.changed() - - def _handle_type2abc(self, node, results, module, abc): - touch_import(None, module, node) - obj = results["obj"] - args = [obj.clone(), String(", " + ".".join([module, abc]))] - return Call(Name("isinstance"), args, prefix=node.prefix) - - def _check_method(self, node, results): - method = getattr(self, "_" + results["method"][0].value) - if isinstance(method, collections.abc.Callable): - if "module" in results: - return method - else: - sub = (str(results["obj"]),) - invocation_str = method.invocation % sub - self.warning(node, "You should use '%s' here." % invocation_str) - return None diff --git a/Lib/lib2to3/fixes/fix_paren.py b/Lib/lib2to3/fixes/fix_paren.py deleted file mode 100644 index df3da5f5232c9c..00000000000000 --- a/Lib/lib2to3/fixes/fix_paren.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Fixer that adds parentheses where they are required - -This converts ``[x for x in 1, 2]`` to ``[x for x in (1, 2)]``.""" - -# By Taek Joo Kim and Benjamin Peterson - -# Local imports -from .. import fixer_base -from ..fixer_util import LParen, RParen - -# XXX This doesn't support nested for loops like [x for x in 1, 2 for x in 1, 2] -class FixParen(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - atom< ('[' | '(') - (listmaker< any - comp_for< - 'for' NAME 'in' - target=testlist_safe< any (',' any)+ [','] - > - [any] - > - > - | - testlist_gexp< any - comp_for< - 'for' NAME 'in' - target=testlist_safe< any (',' any)+ [','] - > - [any] - > - >) - (']' | ')') > - """ - - def transform(self, node, results): - target = results["target"] - - lparen = LParen() - lparen.prefix = target.prefix - target.prefix = "" # Make it hug the parentheses - target.insert_child(0, lparen) - target.append_child(RParen()) diff --git a/Lib/lib2to3/fixes/fix_print.py b/Lib/lib2to3/fixes/fix_print.py deleted file mode 100644 index 8780322265f6fe..00000000000000 --- a/Lib/lib2to3/fixes/fix_print.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for print. - -Change: - 'print' into 'print()' - 'print ...' into 'print(...)' - 'print ... ,' into 'print(..., end=" ")' - 'print >>x, ...' into 'print(..., file=x)' - -No changes are applied if print_function is imported from __future__ - -""" - -# Local imports -from .. import patcomp -from .. import pytree -from ..pgen2 import token -from .. import fixer_base -from ..fixer_util import Name, Call, Comma, String - - -parend_expr = patcomp.compile_pattern( - """atom< '(' [atom|STRING|NAME] ')' >""" - ) - - -class FixPrint(fixer_base.BaseFix): - - BM_compatible = True - - PATTERN = """ - simple_stmt< any* bare='print' any* > | print_stmt - """ - - def transform(self, node, results): - assert results - - bare_print = results.get("bare") - - if bare_print: - # Special-case print all by itself - bare_print.replace(Call(Name("print"), [], - prefix=bare_print.prefix)) - return - assert node.children[0] == Name("print") - args = node.children[1:] - if len(args) == 1 and parend_expr.match(args[0]): - # We don't want to keep sticking parens around an - # already-parenthesised expression. - return - - sep = end = file = None - if args and args[-1] == Comma(): - args = args[:-1] - end = " " - if args and args[0] == pytree.Leaf(token.RIGHTSHIFT, ">>"): - assert len(args) >= 2 - file = args[1].clone() - args = args[3:] # Strip a possible comma after the file expression - # Now synthesize a print(args, sep=..., end=..., file=...) node. - l_args = [arg.clone() for arg in args] - if l_args: - l_args[0].prefix = "" - if sep is not None or end is not None or file is not None: - if sep is not None: - self.add_kwarg(l_args, "sep", String(repr(sep))) - if end is not None: - self.add_kwarg(l_args, "end", String(repr(end))) - if file is not None: - self.add_kwarg(l_args, "file", file) - n_stmt = Call(Name("print"), l_args) - n_stmt.prefix = node.prefix - return n_stmt - - def add_kwarg(self, l_nodes, s_kwd, n_expr): - # XXX All this prefix-setting may lose comments (though rarely) - n_expr.prefix = "" - n_argument = pytree.Node(self.syms.argument, - (Name(s_kwd), - pytree.Leaf(token.EQUAL, "="), - n_expr)) - if l_nodes: - l_nodes.append(Comma()) - n_argument.prefix = " " - l_nodes.append(n_argument) diff --git a/Lib/lib2to3/fixes/fix_raise.py b/Lib/lib2to3/fixes/fix_raise.py deleted file mode 100644 index 05aa21e74a30ff..00000000000000 --- a/Lib/lib2to3/fixes/fix_raise.py +++ /dev/null @@ -1,90 +0,0 @@ -"""Fixer for 'raise E, V, T' - -raise -> raise -raise E -> raise E -raise E, V -> raise E(V) -raise E, V, T -> raise E(V).with_traceback(T) -raise E, None, T -> raise E.with_traceback(T) - -raise (((E, E'), E''), E'''), V -> raise E(V) -raise "foo", V, T -> warns about string exceptions - - -CAVEATS: -1) "raise E, V" will be incorrectly translated if V is an exception - instance. The correct Python 3 idiom is - - raise E from V - - but since we can't detect instance-hood by syntax alone and since - any client code would have to be changed as well, we don't automate - this. -""" -# Author: Collin Winter - -# Local imports -from .. import pytree -from ..pgen2 import token -from .. import fixer_base -from ..fixer_util import Name, Call, Attr, ArgList, is_tuple - -class FixRaise(fixer_base.BaseFix): - - BM_compatible = True - PATTERN = """ - raise_stmt< 'raise' exc=any [',' val=any [',' tb=any]] > - """ - - def transform(self, node, results): - syms = self.syms - - exc = results["exc"].clone() - if exc.type == token.STRING: - msg = "Python 3 does not support string exceptions" - self.cannot_convert(node, msg) - return - - # Python 2 supports - # raise ((((E1, E2), E3), E4), E5), V - # as a synonym for - # raise E1, V - # Since Python 3 will not support this, we recurse down any tuple - # literals, always taking the first element. - if is_tuple(exc): - while is_tuple(exc): - # exc.children[1:-1] is the unparenthesized tuple - # exc.children[1].children[0] is the first element of the tuple - exc = exc.children[1].children[0].clone() - exc.prefix = " " - - if "val" not in results: - # One-argument raise - new = pytree.Node(syms.raise_stmt, [Name("raise"), exc]) - new.prefix = node.prefix - return new - - val = results["val"].clone() - if is_tuple(val): - args = [c.clone() for c in val.children[1:-1]] - else: - val.prefix = "" - args = [val] - - if "tb" in results: - tb = results["tb"].clone() - tb.prefix = "" - - e = exc - # If there's a traceback and None is passed as the value, then don't - # add a call, since the user probably just wants to add a - # traceback. See issue #9661. - if val.type != token.NAME or val.value != "None": - e = Call(exc, args) - with_tb = Attr(e, Name('with_traceback')) + [ArgList([tb])] - new = pytree.Node(syms.simple_stmt, [Name("raise")] + with_tb) - new.prefix = node.prefix - return new - else: - return pytree.Node(syms.raise_stmt, - [Name("raise"), Call(exc, args)], - prefix=node.prefix) diff --git a/Lib/lib2to3/fixes/fix_raw_input.py b/Lib/lib2to3/fixes/fix_raw_input.py deleted file mode 100644 index a51bb694b9e01e..00000000000000 --- a/Lib/lib2to3/fixes/fix_raw_input.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Fixer that changes raw_input(...) into input(...).""" -# Author: Andre Roberge - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - -class FixRawInput(fixer_base.BaseFix): - - BM_compatible = True - PATTERN = """ - power< name='raw_input' trailer< '(' [any] ')' > any* > - """ - - def transform(self, node, results): - name = results["name"] - name.replace(Name("input", prefix=name.prefix)) diff --git a/Lib/lib2to3/fixes/fix_reduce.py b/Lib/lib2to3/fixes/fix_reduce.py deleted file mode 100644 index 00e5aa1c33d482..00000000000000 --- a/Lib/lib2to3/fixes/fix_reduce.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2008 Armin Ronacher. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for reduce(). - -Makes sure reduce() is imported from the functools module if reduce is -used in that module. -""" - -from lib2to3 import fixer_base -from lib2to3.fixer_util import touch_import - - - -class FixReduce(fixer_base.BaseFix): - - BM_compatible = True - order = "pre" - - PATTERN = """ - power< 'reduce' - trailer< '(' - arglist< ( - (not(argument) any ',' - not(argument - > - """ - - def transform(self, node, results): - touch_import('functools', 'reduce', node) diff --git a/Lib/lib2to3/fixes/fix_reload.py b/Lib/lib2to3/fixes/fix_reload.py deleted file mode 100644 index b30841131c51f9..00000000000000 --- a/Lib/lib2to3/fixes/fix_reload.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Fixer for reload(). - -reload(s) -> importlib.reload(s)""" - -# Local imports -from .. import fixer_base -from ..fixer_util import ImportAndCall, touch_import - - -class FixReload(fixer_base.BaseFix): - BM_compatible = True - order = "pre" - - PATTERN = """ - power< 'reload' - trailer< lpar='(' - ( not(arglist | argument) any ','> ) - rpar=')' > - after=any* - > - """ - - def transform(self, node, results): - if results: - # I feel like we should be able to express this logic in the - # PATTERN above but I don't know how to do it so... - obj = results['obj'] - if obj: - if (obj.type == self.syms.argument and - obj.children[0].value in {'**', '*'}): - return # Make no change. - names = ('importlib', 'reload') - new = ImportAndCall(node, results, names) - touch_import(None, 'importlib', node) - return new diff --git a/Lib/lib2to3/fixes/fix_renames.py b/Lib/lib2to3/fixes/fix_renames.py deleted file mode 100644 index c0e3705ab7be19..00000000000000 --- a/Lib/lib2to3/fixes/fix_renames.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Fix incompatible renames - -Fixes: - * sys.maxint -> sys.maxsize -""" -# Author: Christian Heimes -# based on Collin Winter's fix_import - -# Local imports -from .. import fixer_base -from ..fixer_util import Name, attr_chain - -MAPPING = {"sys": {"maxint" : "maxsize"}, - } -LOOKUP = {} - -def alternates(members): - return "(" + "|".join(map(repr, members)) + ")" - - -def build_pattern(): - #bare = set() - for module, replace in list(MAPPING.items()): - for old_attr, new_attr in list(replace.items()): - LOOKUP[(module, old_attr)] = new_attr - #bare.add(module) - #bare.add(old_attr) - #yield """ - # import_name< 'import' (module=%r - # | dotted_as_names< any* module=%r any* >) > - # """ % (module, module) - yield """ - import_from< 'from' module_name=%r 'import' - ( attr_name=%r | import_as_name< attr_name=%r 'as' any >) > - """ % (module, old_attr, old_attr) - yield """ - power< module_name=%r trailer< '.' attr_name=%r > any* > - """ % (module, old_attr) - #yield """bare_name=%s""" % alternates(bare) - - -class FixRenames(fixer_base.BaseFix): - BM_compatible = True - PATTERN = "|".join(build_pattern()) - - order = "pre" # Pre-order tree traversal - - # Don't match the node if it's within another match - def match(self, node): - match = super(FixRenames, self).match - results = match(node) - if results: - if any(match(obj) for obj in attr_chain(node, "parent")): - return False - return results - return False - - #def start_tree(self, tree, filename): - # super(FixRenames, self).start_tree(tree, filename) - # self.replace = {} - - def transform(self, node, results): - mod_name = results.get("module_name") - attr_name = results.get("attr_name") - #bare_name = results.get("bare_name") - #import_mod = results.get("module") - - if mod_name and attr_name: - new_attr = LOOKUP[(mod_name.value, attr_name.value)] - attr_name.replace(Name(new_attr, prefix=attr_name.prefix)) diff --git a/Lib/lib2to3/fixes/fix_repr.py b/Lib/lib2to3/fixes/fix_repr.py deleted file mode 100644 index 1150bb8b9db2af..00000000000000 --- a/Lib/lib2to3/fixes/fix_repr.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer that transforms `xyzzy` into repr(xyzzy).""" - -# Local imports -from .. import fixer_base -from ..fixer_util import Call, Name, parenthesize - - -class FixRepr(fixer_base.BaseFix): - - BM_compatible = True - PATTERN = """ - atom < '`' expr=any '`' > - """ - - def transform(self, node, results): - expr = results["expr"].clone() - - if expr.type == self.syms.testlist1: - expr = parenthesize(expr) - return Call(Name("repr"), [expr], prefix=node.prefix) diff --git a/Lib/lib2to3/fixes/fix_set_literal.py b/Lib/lib2to3/fixes/fix_set_literal.py deleted file mode 100644 index 762550cf73dc0b..00000000000000 --- a/Lib/lib2to3/fixes/fix_set_literal.py +++ /dev/null @@ -1,53 +0,0 @@ -""" -Optional fixer to transform set() calls to set literals. -""" - -# Author: Benjamin Peterson - -from lib2to3 import fixer_base, pytree -from lib2to3.fixer_util import token, syms - - - -class FixSetLiteral(fixer_base.BaseFix): - - BM_compatible = True - explicit = True - - PATTERN = """power< 'set' trailer< '(' - (atom=atom< '[' (items=listmaker< any ((',' any)* [',']) > - | - single=any) ']' > - | - atom< '(' items=testlist_gexp< any ((',' any)* [',']) > ')' > - ) - ')' > > - """ - - def transform(self, node, results): - single = results.get("single") - if single: - # Make a fake listmaker - fake = pytree.Node(syms.listmaker, [single.clone()]) - single.replace(fake) - items = fake - else: - items = results["items"] - - # Build the contents of the literal - literal = [pytree.Leaf(token.LBRACE, "{")] - literal.extend(n.clone() for n in items.children) - literal.append(pytree.Leaf(token.RBRACE, "}")) - # Set the prefix of the right brace to that of the ')' or ']' - literal[-1].prefix = items.next_sibling.prefix - maker = pytree.Node(syms.dictsetmaker, literal) - maker.prefix = node.prefix - - # If the original was a one tuple, we need to remove the extra comma. - if len(maker.children) == 4: - n = maker.children[2] - n.remove() - maker.children[-1].prefix = n.prefix - - # Finally, replace the set call with our shiny new literal. - return maker diff --git a/Lib/lib2to3/fixes/fix_standarderror.py b/Lib/lib2to3/fixes/fix_standarderror.py deleted file mode 100644 index dc742167e6e9d4..00000000000000 --- a/Lib/lib2to3/fixes/fix_standarderror.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2007 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for StandardError -> Exception.""" - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - - -class FixStandarderror(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - 'StandardError' - """ - - def transform(self, node, results): - return Name("Exception", prefix=node.prefix) diff --git a/Lib/lib2to3/fixes/fix_sys_exc.py b/Lib/lib2to3/fixes/fix_sys_exc.py deleted file mode 100644 index f6039690374ab2..00000000000000 --- a/Lib/lib2to3/fixes/fix_sys_exc.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Fixer for sys.exc_{type, value, traceback} - -sys.exc_type -> sys.exc_info()[0] -sys.exc_value -> sys.exc_info()[1] -sys.exc_traceback -> sys.exc_info()[2] -""" - -# By Jeff Balogh and Benjamin Peterson - -# Local imports -from .. import fixer_base -from ..fixer_util import Attr, Call, Name, Number, Subscript, Node, syms - -class FixSysExc(fixer_base.BaseFix): - # This order matches the ordering of sys.exc_info(). - exc_info = ["exc_type", "exc_value", "exc_traceback"] - BM_compatible = True - PATTERN = """ - power< 'sys' trailer< dot='.' attribute=(%s) > > - """ % '|'.join("'%s'" % e for e in exc_info) - - def transform(self, node, results): - sys_attr = results["attribute"][0] - index = Number(self.exc_info.index(sys_attr.value)) - - call = Call(Name("exc_info"), prefix=sys_attr.prefix) - attr = Attr(Name("sys"), call) - attr[1].children[0].prefix = results["dot"].prefix - attr.append(Subscript(index)) - return Node(syms.power, attr, prefix=node.prefix) diff --git a/Lib/lib2to3/fixes/fix_throw.py b/Lib/lib2to3/fixes/fix_throw.py deleted file mode 100644 index aac29169b4e98e..00000000000000 --- a/Lib/lib2to3/fixes/fix_throw.py +++ /dev/null @@ -1,56 +0,0 @@ -"""Fixer for generator.throw(E, V, T). - -g.throw(E) -> g.throw(E) -g.throw(E, V) -> g.throw(E(V)) -g.throw(E, V, T) -> g.throw(E(V).with_traceback(T)) - -g.throw("foo"[, V[, T]]) will warn about string exceptions.""" -# Author: Collin Winter - -# Local imports -from .. import pytree -from ..pgen2 import token -from .. import fixer_base -from ..fixer_util import Name, Call, ArgList, Attr, is_tuple - -class FixThrow(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - power< any trailer< '.' 'throw' > - trailer< '(' args=arglist< exc=any ',' val=any [',' tb=any] > ')' > - > - | - power< any trailer< '.' 'throw' > trailer< '(' exc=any ')' > > - """ - - def transform(self, node, results): - syms = self.syms - - exc = results["exc"].clone() - if exc.type is token.STRING: - self.cannot_convert(node, "Python 3 does not support string exceptions") - return - - # Leave "g.throw(E)" alone - val = results.get("val") - if val is None: - return - - val = val.clone() - if is_tuple(val): - args = [c.clone() for c in val.children[1:-1]] - else: - val.prefix = "" - args = [val] - - throw_args = results["args"] - - if "tb" in results: - tb = results["tb"].clone() - tb.prefix = "" - - e = Call(exc, args) - with_tb = Attr(e, Name('with_traceback')) + [ArgList([tb])] - throw_args.replace(pytree.Node(syms.power, with_tb)) - else: - throw_args.replace(Call(exc, args)) diff --git a/Lib/lib2to3/fixes/fix_tuple_params.py b/Lib/lib2to3/fixes/fix_tuple_params.py deleted file mode 100644 index cad755ffdbefb3..00000000000000 --- a/Lib/lib2to3/fixes/fix_tuple_params.py +++ /dev/null @@ -1,175 +0,0 @@ -"""Fixer for function definitions with tuple parameters. - -def func(((a, b), c), d): - ... - - -> - -def func(x, d): - ((a, b), c) = x - ... - -It will also support lambdas: - - lambda (x, y): x + y -> lambda t: t[0] + t[1] - - # The parens are a syntax error in Python 3 - lambda (x): x + y -> lambda x: x + y -""" -# Author: Collin Winter - -# Local imports -from .. import pytree -from ..pgen2 import token -from .. import fixer_base -from ..fixer_util import Assign, Name, Newline, Number, Subscript, syms - -def is_docstring(stmt): - return isinstance(stmt, pytree.Node) and \ - stmt.children[0].type == token.STRING - -class FixTupleParams(fixer_base.BaseFix): - run_order = 4 #use a lower order since lambda is part of other - #patterns - BM_compatible = True - - PATTERN = """ - funcdef< 'def' any parameters< '(' args=any ')' > - ['->' any] ':' suite=any+ > - | - lambda= - lambdef< 'lambda' args=vfpdef< '(' inner=any ')' > - ':' body=any - > - """ - - def transform(self, node, results): - if "lambda" in results: - return self.transform_lambda(node, results) - - new_lines = [] - suite = results["suite"] - args = results["args"] - # This crap is so "def foo(...): x = 5; y = 7" is handled correctly. - # TODO(cwinter): suite-cleanup - if suite[0].children[1].type == token.INDENT: - start = 2 - indent = suite[0].children[1].value - end = Newline() - else: - start = 0 - indent = "; " - end = pytree.Leaf(token.INDENT, "") - - # We need access to self for new_name(), and making this a method - # doesn't feel right. Closing over self and new_lines makes the - # code below cleaner. - def handle_tuple(tuple_arg, add_prefix=False): - n = Name(self.new_name()) - arg = tuple_arg.clone() - arg.prefix = "" - stmt = Assign(arg, n.clone()) - if add_prefix: - n.prefix = " " - tuple_arg.replace(n) - new_lines.append(pytree.Node(syms.simple_stmt, - [stmt, end.clone()])) - - if args.type == syms.tfpdef: - handle_tuple(args) - elif args.type == syms.typedargslist: - for i, arg in enumerate(args.children): - if arg.type == syms.tfpdef: - # Without add_prefix, the emitted code is correct, - # just ugly. - handle_tuple(arg, add_prefix=(i > 0)) - - if not new_lines: - return - - # This isn't strictly necessary, but it plays nicely with other fixers. - # TODO(cwinter) get rid of this when children becomes a smart list - for line in new_lines: - line.parent = suite[0] - - # TODO(cwinter) suite-cleanup - after = start - if start == 0: - new_lines[0].prefix = " " - elif is_docstring(suite[0].children[start]): - new_lines[0].prefix = indent - after = start + 1 - - for line in new_lines: - line.parent = suite[0] - suite[0].children[after:after] = new_lines - for i in range(after+1, after+len(new_lines)+1): - suite[0].children[i].prefix = indent - suite[0].changed() - - def transform_lambda(self, node, results): - args = results["args"] - body = results["body"] - inner = simplify_args(results["inner"]) - - # Replace lambda ((((x)))): x with lambda x: x - if inner.type == token.NAME: - inner = inner.clone() - inner.prefix = " " - args.replace(inner) - return - - params = find_params(args) - to_index = map_to_index(params) - tup_name = self.new_name(tuple_name(params)) - - new_param = Name(tup_name, prefix=" ") - args.replace(new_param.clone()) - for n in body.post_order(): - if n.type == token.NAME and n.value in to_index: - subscripts = [c.clone() for c in to_index[n.value]] - new = pytree.Node(syms.power, - [new_param.clone()] + subscripts) - new.prefix = n.prefix - n.replace(new) - - -### Helper functions for transform_lambda() - -def simplify_args(node): - if node.type in (syms.vfplist, token.NAME): - return node - elif node.type == syms.vfpdef: - # These look like vfpdef< '(' x ')' > where x is NAME - # or another vfpdef instance (leading to recursion). - while node.type == syms.vfpdef: - node = node.children[1] - return node - raise RuntimeError("Received unexpected node %s" % node) - -def find_params(node): - if node.type == syms.vfpdef: - return find_params(node.children[1]) - elif node.type == token.NAME: - return node.value - return [find_params(c) for c in node.children if c.type != token.COMMA] - -def map_to_index(param_list, prefix=[], d=None): - if d is None: - d = {} - for i, obj in enumerate(param_list): - trailer = [Subscript(Number(str(i)))] - if isinstance(obj, list): - map_to_index(obj, trailer, d=d) - else: - d[obj] = prefix + trailer - return d - -def tuple_name(param_list): - l = [] - for obj in param_list: - if isinstance(obj, list): - l.append(tuple_name(obj)) - else: - l.append(obj) - return "_".join(l) diff --git a/Lib/lib2to3/fixes/fix_types.py b/Lib/lib2to3/fixes/fix_types.py deleted file mode 100644 index 67bf51f2f5b85a..00000000000000 --- a/Lib/lib2to3/fixes/fix_types.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2007 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for removing uses of the types module. - -These work for only the known names in the types module. The forms above -can include types. or not. ie, It is assumed the module is imported either as: - - import types - from types import ... # either * or specific types - -The import statements are not modified. - -There should be another fixer that handles at least the following constants: - - type([]) -> list - type(()) -> tuple - type('') -> str - -""" - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - -_TYPE_MAPPING = { - 'BooleanType' : 'bool', - 'BufferType' : 'memoryview', - 'ClassType' : 'type', - 'ComplexType' : 'complex', - 'DictType': 'dict', - 'DictionaryType' : 'dict', - 'EllipsisType' : 'type(Ellipsis)', - #'FileType' : 'io.IOBase', - 'FloatType': 'float', - 'IntType': 'int', - 'ListType': 'list', - 'LongType': 'int', - 'ObjectType' : 'object', - 'NoneType': 'type(None)', - 'NotImplementedType' : 'type(NotImplemented)', - 'SliceType' : 'slice', - 'StringType': 'bytes', # XXX ? - 'StringTypes' : '(str,)', # XXX ? - 'TupleType': 'tuple', - 'TypeType' : 'type', - 'UnicodeType': 'str', - 'XRangeType' : 'range', - } - -_pats = ["power< 'types' trailer< '.' name='%s' > >" % t for t in _TYPE_MAPPING] - -class FixTypes(fixer_base.BaseFix): - BM_compatible = True - PATTERN = '|'.join(_pats) - - def transform(self, node, results): - new_value = _TYPE_MAPPING.get(results["name"].value) - if new_value: - return Name(new_value, prefix=node.prefix) - return None diff --git a/Lib/lib2to3/fixes/fix_unicode.py b/Lib/lib2to3/fixes/fix_unicode.py deleted file mode 100644 index c7982c2b97c3e1..00000000000000 --- a/Lib/lib2to3/fixes/fix_unicode.py +++ /dev/null @@ -1,42 +0,0 @@ -r"""Fixer for unicode. - -* Changes unicode to str and unichr to chr. - -* If "...\u..." is not unicode literal change it into "...\\u...". - -* Change u"..." into "...". - -""" - -from ..pgen2 import token -from .. import fixer_base - -_mapping = {"unichr" : "chr", "unicode" : "str"} - -class FixUnicode(fixer_base.BaseFix): - BM_compatible = True - PATTERN = "STRING | 'unicode' | 'unichr'" - - def start_tree(self, tree, filename): - super(FixUnicode, self).start_tree(tree, filename) - self.unicode_literals = 'unicode_literals' in tree.future_features - - def transform(self, node, results): - if node.type == token.NAME: - new = node.clone() - new.value = _mapping[node.value] - return new - elif node.type == token.STRING: - val = node.value - if not self.unicode_literals and val[0] in '\'"' and '\\' in val: - val = r'\\'.join([ - v.replace('\\u', r'\\u').replace('\\U', r'\\U') - for v in val.split(r'\\') - ]) - if val[0] in 'uU': - val = val[1:] - if val == node.value: - return node - new = node.clone() - new.value = val - return new diff --git a/Lib/lib2to3/fixes/fix_urllib.py b/Lib/lib2to3/fixes/fix_urllib.py deleted file mode 100644 index ab892bc52494c2..00000000000000 --- a/Lib/lib2to3/fixes/fix_urllib.py +++ /dev/null @@ -1,196 +0,0 @@ -"""Fix changes imports of urllib which are now incompatible. - This is rather similar to fix_imports, but because of the more - complex nature of the fixing for urllib, it has its own fixer. -""" -# Author: Nick Edds - -# Local imports -from lib2to3.fixes.fix_imports import alternates, FixImports -from lib2to3.fixer_util import (Name, Comma, FromImport, Newline, - find_indentation, Node, syms) - -MAPPING = {"urllib": [ - ("urllib.request", - ["URLopener", "FancyURLopener", "urlretrieve", - "_urlopener", "urlopen", "urlcleanup", - "pathname2url", "url2pathname", "getproxies"]), - ("urllib.parse", - ["quote", "quote_plus", "unquote", "unquote_plus", - "urlencode", "splitattr", "splithost", "splitnport", - "splitpasswd", "splitport", "splitquery", "splittag", - "splittype", "splituser", "splitvalue", ]), - ("urllib.error", - ["ContentTooShortError"])], - "urllib2" : [ - ("urllib.request", - ["urlopen", "install_opener", "build_opener", - "Request", "OpenerDirector", "BaseHandler", - "HTTPDefaultErrorHandler", "HTTPRedirectHandler", - "HTTPCookieProcessor", "ProxyHandler", - "HTTPPasswordMgr", - "HTTPPasswordMgrWithDefaultRealm", - "AbstractBasicAuthHandler", - "HTTPBasicAuthHandler", "ProxyBasicAuthHandler", - "AbstractDigestAuthHandler", - "HTTPDigestAuthHandler", "ProxyDigestAuthHandler", - "HTTPHandler", "HTTPSHandler", "FileHandler", - "FTPHandler", "CacheFTPHandler", - "UnknownHandler"]), - ("urllib.error", - ["URLError", "HTTPError"]), - ] -} - -# Duplicate the url parsing functions for urllib2. -MAPPING["urllib2"].append(MAPPING["urllib"][1]) - - -def build_pattern(): - bare = set() - for old_module, changes in MAPPING.items(): - for change in changes: - new_module, members = change - members = alternates(members) - yield """import_name< 'import' (module=%r - | dotted_as_names< any* module=%r any* >) > - """ % (old_module, old_module) - yield """import_from< 'from' mod_member=%r 'import' - ( member=%s | import_as_name< member=%s 'as' any > | - import_as_names< members=any* >) > - """ % (old_module, members, members) - yield """import_from< 'from' module_star=%r 'import' star='*' > - """ % old_module - yield """import_name< 'import' - dotted_as_name< module_as=%r 'as' any > > - """ % old_module - # bare_with_attr has a special significance for FixImports.match(). - yield """power< bare_with_attr=%r trailer< '.' member=%s > any* > - """ % (old_module, members) - - -class FixUrllib(FixImports): - - def build_pattern(self): - return "|".join(build_pattern()) - - def transform_import(self, node, results): - """Transform for the basic import case. Replaces the old - import name with a comma separated list of its - replacements. - """ - import_mod = results.get("module") - pref = import_mod.prefix - - names = [] - - # create a Node list of the replacement modules - for name in MAPPING[import_mod.value][:-1]: - names.extend([Name(name[0], prefix=pref), Comma()]) - names.append(Name(MAPPING[import_mod.value][-1][0], prefix=pref)) - import_mod.replace(names) - - def transform_member(self, node, results): - """Transform for imports of specific module elements. Replaces - the module to be imported from with the appropriate new - module. - """ - mod_member = results.get("mod_member") - pref = mod_member.prefix - member = results.get("member") - - # Simple case with only a single member being imported - if member: - # this may be a list of length one, or just a node - if isinstance(member, list): - member = member[0] - new_name = None - for change in MAPPING[mod_member.value]: - if member.value in change[1]: - new_name = change[0] - break - if new_name: - mod_member.replace(Name(new_name, prefix=pref)) - else: - self.cannot_convert(node, "This is an invalid module element") - - # Multiple members being imported - else: - # a dictionary for replacements, order matters - modules = [] - mod_dict = {} - members = results["members"] - for member in members: - # we only care about the actual members - if member.type == syms.import_as_name: - as_name = member.children[2].value - member_name = member.children[0].value - else: - member_name = member.value - as_name = None - if member_name != ",": - for change in MAPPING[mod_member.value]: - if member_name in change[1]: - if change[0] not in mod_dict: - modules.append(change[0]) - mod_dict.setdefault(change[0], []).append(member) - - new_nodes = [] - indentation = find_indentation(node) - first = True - def handle_name(name, prefix): - if name.type == syms.import_as_name: - kids = [Name(name.children[0].value, prefix=prefix), - name.children[1].clone(), - name.children[2].clone()] - return [Node(syms.import_as_name, kids)] - return [Name(name.value, prefix=prefix)] - for module in modules: - elts = mod_dict[module] - names = [] - for elt in elts[:-1]: - names.extend(handle_name(elt, pref)) - names.append(Comma()) - names.extend(handle_name(elts[-1], pref)) - new = FromImport(module, names) - if not first or node.parent.prefix.endswith(indentation): - new.prefix = indentation - new_nodes.append(new) - first = False - if new_nodes: - nodes = [] - for new_node in new_nodes[:-1]: - nodes.extend([new_node, Newline()]) - nodes.append(new_nodes[-1]) - node.replace(nodes) - else: - self.cannot_convert(node, "All module elements are invalid") - - def transform_dot(self, node, results): - """Transform for calls to module members in code.""" - module_dot = results.get("bare_with_attr") - member = results.get("member") - new_name = None - if isinstance(member, list): - member = member[0] - for change in MAPPING[module_dot.value]: - if member.value in change[1]: - new_name = change[0] - break - if new_name: - module_dot.replace(Name(new_name, - prefix=module_dot.prefix)) - else: - self.cannot_convert(node, "This is an invalid module element") - - def transform(self, node, results): - if results.get("module"): - self.transform_import(node, results) - elif results.get("mod_member"): - self.transform_member(node, results) - elif results.get("bare_with_attr"): - self.transform_dot(node, results) - # Renaming and star imports are not supported for these modules. - elif results.get("module_star"): - self.cannot_convert(node, "Cannot handle star imports.") - elif results.get("module_as"): - self.cannot_convert(node, "This module is now multiple modules") diff --git a/Lib/lib2to3/fixes/fix_ws_comma.py b/Lib/lib2to3/fixes/fix_ws_comma.py deleted file mode 100644 index a54a376c472afb..00000000000000 --- a/Lib/lib2to3/fixes/fix_ws_comma.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Fixer that changes 'a ,b' into 'a, b'. - -This also changes '{a :b}' into '{a: b}', but does not touch other -uses of colons. It does not touch other uses of whitespace. - -""" - -from .. import pytree -from ..pgen2 import token -from .. import fixer_base - -class FixWsComma(fixer_base.BaseFix): - - explicit = True # The user must ask for this fixers - - PATTERN = """ - any<(not(',') any)+ ',' ((not(',') any)+ ',')* [not(',') any]> - """ - - COMMA = pytree.Leaf(token.COMMA, ",") - COLON = pytree.Leaf(token.COLON, ":") - SEPS = (COMMA, COLON) - - def transform(self, node, results): - new = node.clone() - comma = False - for child in new.children: - if child in self.SEPS: - prefix = child.prefix - if prefix.isspace() and "\n" not in prefix: - child.prefix = "" - comma = True - else: - if comma: - prefix = child.prefix - if not prefix: - child.prefix = " " - comma = False - return new diff --git a/Lib/lib2to3/fixes/fix_xrange.py b/Lib/lib2to3/fixes/fix_xrange.py deleted file mode 100644 index 1e491e166a3f1c..00000000000000 --- a/Lib/lib2to3/fixes/fix_xrange.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2007 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer that changes xrange(...) into range(...).""" - -# Local imports -from .. import fixer_base -from ..fixer_util import Name, Call, consuming_calls -from .. import patcomp - - -class FixXrange(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - power< - (name='range'|name='xrange') trailer< '(' args=any ')' > - rest=any* > - """ - - def start_tree(self, tree, filename): - super(FixXrange, self).start_tree(tree, filename) - self.transformed_xranges = set() - - def finish_tree(self, tree, filename): - self.transformed_xranges = None - - def transform(self, node, results): - name = results["name"] - if name.value == "xrange": - return self.transform_xrange(node, results) - elif name.value == "range": - return self.transform_range(node, results) - else: - raise ValueError(repr(name)) - - def transform_xrange(self, node, results): - name = results["name"] - name.replace(Name("range", prefix=name.prefix)) - # This prevents the new range call from being wrapped in a list later. - self.transformed_xranges.add(id(node)) - - def transform_range(self, node, results): - if (id(node) not in self.transformed_xranges and - not self.in_special_context(node)): - range_call = Call(Name("range"), [results["args"].clone()]) - # Encase the range call in list(). - list_call = Call(Name("list"), [range_call], - prefix=node.prefix) - # Put things that were after the range() call after the list call. - for n in results["rest"]: - list_call.append_child(n) - return list_call - - P1 = "power< func=NAME trailer< '(' node=any ')' > any* >" - p1 = patcomp.compile_pattern(P1) - - P2 = """for_stmt< 'for' any 'in' node=any ':' any* > - | comp_for< 'for' any 'in' node=any any* > - | comparison< any 'in' node=any any*> - """ - p2 = patcomp.compile_pattern(P2) - - def in_special_context(self, node): - if node.parent is None: - return False - results = {} - if (node.parent.parent is not None and - self.p1.match(node.parent.parent, results) and - results["node"] is node): - # list(d.keys()) -> list(d.keys()), etc. - return results["func"].value in consuming_calls - # for ... in d.iterkeys() -> for ... in d.keys(), etc. - return self.p2.match(node.parent, results) and results["node"] is node diff --git a/Lib/lib2to3/fixes/fix_xreadlines.py b/Lib/lib2to3/fixes/fix_xreadlines.py deleted file mode 100644 index 3e3f71ab045573..00000000000000 --- a/Lib/lib2to3/fixes/fix_xreadlines.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Fix "for x in f.xreadlines()" -> "for x in f". - -This fixer will also convert g(f.xreadlines) into g(f.__iter__).""" -# Author: Collin Winter - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - - -class FixXreadlines(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - power< call=any+ trailer< '.' 'xreadlines' > trailer< '(' ')' > > - | - power< any+ trailer< '.' no_call='xreadlines' > > - """ - - def transform(self, node, results): - no_call = results.get("no_call") - - if no_call: - no_call.replace(Name("__iter__", prefix=no_call.prefix)) - else: - node.replace([x.clone() for x in results["call"]]) diff --git a/Lib/lib2to3/fixes/fix_zip.py b/Lib/lib2to3/fixes/fix_zip.py deleted file mode 100644 index 52c28df6aab411..00000000000000 --- a/Lib/lib2to3/fixes/fix_zip.py +++ /dev/null @@ -1,46 +0,0 @@ -""" -Fixer that changes zip(seq0, seq1, ...) into list(zip(seq0, seq1, ...) -unless there exists a 'from future_builtins import zip' statement in the -top-level namespace. - -We avoid the transformation if the zip() call is directly contained in -iter(<>), list(<>), tuple(<>), sorted(<>), ...join(<>), or for V in <>:. -""" - -# Local imports -from .. import fixer_base -from ..pytree import Node -from ..pygram import python_symbols as syms -from ..fixer_util import Name, ArgList, in_special_context - - -class FixZip(fixer_base.ConditionalFix): - - BM_compatible = True - PATTERN = """ - power< 'zip' args=trailer< '(' [any] ')' > [trailers=trailer*] - > - """ - - skip_on = "future_builtins.zip" - - def transform(self, node, results): - if self.should_skip(node): - return - - if in_special_context(node): - return None - - args = results['args'].clone() - args.prefix = "" - - trailers = [] - if 'trailers' in results: - trailers = [n.clone() for n in results['trailers']] - for n in trailers: - n.prefix = "" - - new = Node(syms.power, [Name("zip"), args], prefix="") - new = Node(syms.power, [Name("list"), ArgList([new])] + trailers) - new.prefix = node.prefix - return new diff --git a/Lib/lib2to3/main.py b/Lib/lib2to3/main.py deleted file mode 100644 index f2849fd6be3fd9..00000000000000 --- a/Lib/lib2to3/main.py +++ /dev/null @@ -1,273 +0,0 @@ -""" -Main program for 2to3. -""" - -from __future__ import with_statement, print_function - -import sys -import os -import difflib -import logging -import shutil -import optparse - -from . import refactor - - -def diff_texts(a, b, filename): - """Return a unified diff of two strings.""" - a = a.splitlines() - b = b.splitlines() - return difflib.unified_diff(a, b, filename, filename, - "(original)", "(refactored)", - lineterm="") - - -class StdoutRefactoringTool(refactor.MultiprocessRefactoringTool): - """ - A refactoring tool that can avoid overwriting its input files. - Prints output to stdout. - - Output files can optionally be written to a different directory and or - have an extra file suffix appended to their name for use in situations - where you do not want to replace the input files. - """ - - def __init__(self, fixers, options, explicit, nobackups, show_diffs, - input_base_dir='', output_dir='', append_suffix=''): - """ - Args: - fixers: A list of fixers to import. - options: A dict with RefactoringTool configuration. - explicit: A list of fixers to run even if they are explicit. - nobackups: If true no backup '.bak' files will be created for those - files that are being refactored. - show_diffs: Should diffs of the refactoring be printed to stdout? - input_base_dir: The base directory for all input files. This class - will strip this path prefix off of filenames before substituting - it with output_dir. Only meaningful if output_dir is supplied. - All files processed by refactor() must start with this path. - output_dir: If supplied, all converted files will be written into - this directory tree instead of input_base_dir. - append_suffix: If supplied, all files output by this tool will have - this appended to their filename. Useful for changing .py to - .py3 for example by passing append_suffix='3'. - """ - self.nobackups = nobackups - self.show_diffs = show_diffs - if input_base_dir and not input_base_dir.endswith(os.sep): - input_base_dir += os.sep - self._input_base_dir = input_base_dir - self._output_dir = output_dir - self._append_suffix = append_suffix - super(StdoutRefactoringTool, self).__init__(fixers, options, explicit) - - def log_error(self, msg, *args, **kwargs): - self.errors.append((msg, args, kwargs)) - self.logger.error(msg, *args, **kwargs) - - def write_file(self, new_text, filename, old_text, encoding): - orig_filename = filename - if self._output_dir: - if filename.startswith(self._input_base_dir): - filename = os.path.join(self._output_dir, - filename[len(self._input_base_dir):]) - else: - raise ValueError('filename %s does not start with the ' - 'input_base_dir %s' % ( - filename, self._input_base_dir)) - if self._append_suffix: - filename += self._append_suffix - if orig_filename != filename: - output_dir = os.path.dirname(filename) - if not os.path.isdir(output_dir) and output_dir: - os.makedirs(output_dir) - self.log_message('Writing converted %s to %s.', orig_filename, - filename) - if not self.nobackups: - # Make backup - backup = filename + ".bak" - if os.path.lexists(backup): - try: - os.remove(backup) - except OSError: - self.log_message("Can't remove backup %s", backup) - try: - os.rename(filename, backup) - except OSError: - self.log_message("Can't rename %s to %s", filename, backup) - # Actually write the new file - write = super(StdoutRefactoringTool, self).write_file - write(new_text, filename, old_text, encoding) - if not self.nobackups: - shutil.copymode(backup, filename) - if orig_filename != filename: - # Preserve the file mode in the new output directory. - shutil.copymode(orig_filename, filename) - - def print_output(self, old, new, filename, equal): - if equal: - self.log_message("No changes to %s", filename) - else: - self.log_message("Refactored %s", filename) - if self.show_diffs: - diff_lines = diff_texts(old, new, filename) - try: - if self.output_lock is not None: - with self.output_lock: - for line in diff_lines: - print(line) - sys.stdout.flush() - else: - for line in diff_lines: - print(line) - except UnicodeEncodeError: - warn("couldn't encode %s's diff for your terminal" % - (filename,)) - return - -def warn(msg): - print("WARNING: %s" % (msg,), file=sys.stderr) - - -def main(fixer_pkg, args=None): - """Main program. - - Args: - fixer_pkg: the name of a package where the fixers are located. - args: optional; a list of command line arguments. If omitted, - sys.argv[1:] is used. - - Returns a suggested exit status (0, 1, 2). - """ - # Set up option parser - parser = optparse.OptionParser(usage="2to3 [options] file|dir ...") - parser.add_option("-d", "--doctests_only", action="store_true", - help="Fix up doctests only") - parser.add_option("-f", "--fix", action="append", default=[], - help="Each FIX specifies a transformation; default: all") - parser.add_option("-j", "--processes", action="store", default=1, - type="int", help="Run 2to3 concurrently") - parser.add_option("-x", "--nofix", action="append", default=[], - help="Prevent a transformation from being run") - parser.add_option("-l", "--list-fixes", action="store_true", - help="List available transformations") - parser.add_option("-p", "--print-function", action="store_true", - help="Modify the grammar so that print() is a function") - parser.add_option("-e", "--exec-function", action="store_true", - help="Modify the grammar so that exec() is a function") - parser.add_option("-v", "--verbose", action="store_true", - help="More verbose logging") - parser.add_option("--no-diffs", action="store_true", - help="Don't show diffs of the refactoring") - parser.add_option("-w", "--write", action="store_true", - help="Write back modified files") - parser.add_option("-n", "--nobackups", action="store_true", default=False, - help="Don't write backups for modified files") - parser.add_option("-o", "--output-dir", action="store", type="str", - default="", help="Put output files in this directory " - "instead of overwriting the input files. Requires -n.") - parser.add_option("-W", "--write-unchanged-files", action="store_true", - help="Also write files even if no changes were required" - " (useful with --output-dir); implies -w.") - parser.add_option("--add-suffix", action="store", type="str", default="", - help="Append this string to all output filenames." - " Requires -n if non-empty. " - "ex: --add-suffix='3' will generate .py3 files.") - - # Parse command line arguments - refactor_stdin = False - flags = {} - options, args = parser.parse_args(args) - if options.write_unchanged_files: - flags["write_unchanged_files"] = True - if not options.write: - warn("--write-unchanged-files/-W implies -w.") - options.write = True - # If we allowed these, the original files would be renamed to backup names - # but not replaced. - if options.output_dir and not options.nobackups: - parser.error("Can't use --output-dir/-o without -n.") - if options.add_suffix and not options.nobackups: - parser.error("Can't use --add-suffix without -n.") - - if not options.write and options.no_diffs: - warn("not writing files and not printing diffs; that's not very useful") - if not options.write and options.nobackups: - parser.error("Can't use -n without -w") - if options.list_fixes: - print("Available transformations for the -f/--fix option:") - for fixname in refactor.get_all_fix_names(fixer_pkg): - print(fixname) - if not args: - return 0 - if not args: - print("At least one file or directory argument required.", file=sys.stderr) - print("Use --help to show usage.", file=sys.stderr) - return 2 - if "-" in args: - refactor_stdin = True - if options.write: - print("Can't write to stdin.", file=sys.stderr) - return 2 - if options.print_function: - flags["print_function"] = True - - if options.exec_function: - flags["exec_function"] = True - - # Set up logging handler - level = logging.DEBUG if options.verbose else logging.INFO - logging.basicConfig(format='%(name)s: %(message)s', level=level) - logger = logging.getLogger('lib2to3.main') - - # Initialize the refactoring tool - avail_fixes = set(refactor.get_fixers_from_package(fixer_pkg)) - unwanted_fixes = set(fixer_pkg + ".fix_" + fix for fix in options.nofix) - explicit = set() - if options.fix: - all_present = False - for fix in options.fix: - if fix == "all": - all_present = True - else: - explicit.add(fixer_pkg + ".fix_" + fix) - requested = avail_fixes.union(explicit) if all_present else explicit - else: - requested = avail_fixes.union(explicit) - fixer_names = requested.difference(unwanted_fixes) - input_base_dir = os.path.commonprefix(args) - if (input_base_dir and not input_base_dir.endswith(os.sep) - and not os.path.isdir(input_base_dir)): - # One or more similar names were passed, their directory is the base. - # os.path.commonprefix() is ignorant of path elements, this corrects - # for that weird API. - input_base_dir = os.path.dirname(input_base_dir) - if options.output_dir: - input_base_dir = input_base_dir.rstrip(os.sep) - logger.info('Output in %r will mirror the input directory %r layout.', - options.output_dir, input_base_dir) - rt = StdoutRefactoringTool( - sorted(fixer_names), flags, sorted(explicit), - options.nobackups, not options.no_diffs, - input_base_dir=input_base_dir, - output_dir=options.output_dir, - append_suffix=options.add_suffix) - - # Refactor all files and directories passed as arguments - if not rt.errors: - if refactor_stdin: - rt.refactor_stdin() - else: - try: - rt.refactor(args, options.write, options.doctests_only, - options.processes) - except refactor.MultiprocessingUnsupported: - assert options.processes > 1 - print("Sorry, -j isn't supported on this platform.", - file=sys.stderr) - return 1 - rt.summarize() - - # Return error status (0 if rt.errors is zero) - return int(bool(rt.errors)) diff --git a/Lib/lib2to3/patcomp.py b/Lib/lib2to3/patcomp.py deleted file mode 100644 index f57f4954b26ce7..00000000000000 --- a/Lib/lib2to3/patcomp.py +++ /dev/null @@ -1,204 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Pattern compiler. - -The grammar is taken from PatternGrammar.txt. - -The compiler compiles a pattern to a pytree.*Pattern instance. -""" - -__author__ = "Guido van Rossum " - -# Python imports -import io - -# Fairly local imports -from .pgen2 import driver, literals, token, tokenize, parse, grammar - -# Really local imports -from . import pytree -from . import pygram - - -class PatternSyntaxError(Exception): - pass - - -def tokenize_wrapper(input): - """Tokenizes a string suppressing significant whitespace.""" - skip = {token.NEWLINE, token.INDENT, token.DEDENT} - tokens = tokenize.generate_tokens(io.StringIO(input).readline) - for quintuple in tokens: - type, value, start, end, line_text = quintuple - if type not in skip: - yield quintuple - - -class PatternCompiler(object): - - def __init__(self, grammar_file=None): - """Initializer. - - Takes an optional alternative filename for the pattern grammar. - """ - if grammar_file is None: - self.grammar = pygram.pattern_grammar - self.syms = pygram.pattern_symbols - else: - self.grammar = driver.load_grammar(grammar_file) - self.syms = pygram.Symbols(self.grammar) - self.pygrammar = pygram.python_grammar - self.pysyms = pygram.python_symbols - self.driver = driver.Driver(self.grammar, convert=pattern_convert) - - def compile_pattern(self, input, debug=False, with_tree=False): - """Compiles a pattern string to a nested pytree.*Pattern object.""" - tokens = tokenize_wrapper(input) - try: - root = self.driver.parse_tokens(tokens, debug=debug) - except parse.ParseError as e: - raise PatternSyntaxError(str(e)) from None - if with_tree: - return self.compile_node(root), root - else: - return self.compile_node(root) - - def compile_node(self, node): - """Compiles a node, recursively. - - This is one big switch on the node type. - """ - # XXX Optimize certain Wildcard-containing-Wildcard patterns - # that can be merged - if node.type == self.syms.Matcher: - node = node.children[0] # Avoid unneeded recursion - - if node.type == self.syms.Alternatives: - # Skip the odd children since they are just '|' tokens - alts = [self.compile_node(ch) for ch in node.children[::2]] - if len(alts) == 1: - return alts[0] - p = pytree.WildcardPattern([[a] for a in alts], min=1, max=1) - return p.optimize() - - if node.type == self.syms.Alternative: - units = [self.compile_node(ch) for ch in node.children] - if len(units) == 1: - return units[0] - p = pytree.WildcardPattern([units], min=1, max=1) - return p.optimize() - - if node.type == self.syms.NegatedUnit: - pattern = self.compile_basic(node.children[1:]) - p = pytree.NegatedPattern(pattern) - return p.optimize() - - assert node.type == self.syms.Unit - - name = None - nodes = node.children - if len(nodes) >= 3 and nodes[1].type == token.EQUAL: - name = nodes[0].value - nodes = nodes[2:] - repeat = None - if len(nodes) >= 2 and nodes[-1].type == self.syms.Repeater: - repeat = nodes[-1] - nodes = nodes[:-1] - - # Now we've reduced it to: STRING | NAME [Details] | (...) | [...] - pattern = self.compile_basic(nodes, repeat) - - if repeat is not None: - assert repeat.type == self.syms.Repeater - children = repeat.children - child = children[0] - if child.type == token.STAR: - min = 0 - max = pytree.HUGE - elif child.type == token.PLUS: - min = 1 - max = pytree.HUGE - elif child.type == token.LBRACE: - assert children[-1].type == token.RBRACE - assert len(children) in (3, 5) - min = max = self.get_int(children[1]) - if len(children) == 5: - max = self.get_int(children[3]) - else: - assert False - if min != 1 or max != 1: - pattern = pattern.optimize() - pattern = pytree.WildcardPattern([[pattern]], min=min, max=max) - - if name is not None: - pattern.name = name - return pattern.optimize() - - def compile_basic(self, nodes, repeat=None): - # Compile STRING | NAME [Details] | (...) | [...] - assert len(nodes) >= 1 - node = nodes[0] - if node.type == token.STRING: - value = str(literals.evalString(node.value)) - return pytree.LeafPattern(_type_of_literal(value), value) - elif node.type == token.NAME: - value = node.value - if value.isupper(): - if value not in TOKEN_MAP: - raise PatternSyntaxError("Invalid token: %r" % value) - if nodes[1:]: - raise PatternSyntaxError("Can't have details for token") - return pytree.LeafPattern(TOKEN_MAP[value]) - else: - if value == "any": - type = None - elif not value.startswith("_"): - type = getattr(self.pysyms, value, None) - if type is None: - raise PatternSyntaxError("Invalid symbol: %r" % value) - if nodes[1:]: # Details present - content = [self.compile_node(nodes[1].children[1])] - else: - content = None - return pytree.NodePattern(type, content) - elif node.value == "(": - return self.compile_node(nodes[1]) - elif node.value == "[": - assert repeat is None - subpattern = self.compile_node(nodes[1]) - return pytree.WildcardPattern([[subpattern]], min=0, max=1) - assert False, node - - def get_int(self, node): - assert node.type == token.NUMBER - return int(node.value) - - -# Map named tokens to the type value for a LeafPattern -TOKEN_MAP = {"NAME": token.NAME, - "STRING": token.STRING, - "NUMBER": token.NUMBER, - "TOKEN": None} - - -def _type_of_literal(value): - if value[0].isalpha(): - return token.NAME - elif value in grammar.opmap: - return grammar.opmap[value] - else: - return None - - -def pattern_convert(grammar, raw_node_info): - """Converts raw node information to a Node or Leaf instance.""" - type, value, context, children = raw_node_info - if children or type in grammar.number2symbol: - return pytree.Node(type, children, context=context) - else: - return pytree.Leaf(type, value, context=context) - - -def compile_pattern(pattern): - return PatternCompiler().compile_pattern(pattern) diff --git a/Lib/lib2to3/pgen2/__init__.py b/Lib/lib2to3/pgen2/__init__.py deleted file mode 100644 index af390484528d87..00000000000000 --- a/Lib/lib2to3/pgen2/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""The pgen2 package.""" diff --git a/Lib/lib2to3/pgen2/conv.py b/Lib/lib2to3/pgen2/conv.py deleted file mode 100644 index ed0cac532e4249..00000000000000 --- a/Lib/lib2to3/pgen2/conv.py +++ /dev/null @@ -1,257 +0,0 @@ -# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Convert graminit.[ch] spit out by pgen to Python code. - -Pgen is the Python parser generator. It is useful to quickly create a -parser from a grammar file in Python's grammar notation. But I don't -want my parsers to be written in C (yet), so I'm translating the -parsing tables to Python data structures and writing a Python parse -engine. - -Note that the token numbers are constants determined by the standard -Python tokenizer. The standard token module defines these numbers and -their names (the names are not used much). The token numbers are -hardcoded into the Python tokenizer and into pgen. A Python -implementation of the Python tokenizer is also available, in the -standard tokenize module. - -On the other hand, symbol numbers (representing the grammar's -non-terminals) are assigned by pgen based on the actual grammar -input. - -Note: this module is pretty much obsolete; the pgen module generates -equivalent grammar tables directly from the Grammar.txt input file -without having to invoke the Python pgen C program. - -""" - -# Python imports -import re - -# Local imports -from pgen2 import grammar, token - - -class Converter(grammar.Grammar): - """Grammar subclass that reads classic pgen output files. - - The run() method reads the tables as produced by the pgen parser - generator, typically contained in two C files, graminit.h and - graminit.c. The other methods are for internal use only. - - See the base class for more documentation. - - """ - - def run(self, graminit_h, graminit_c): - """Load the grammar tables from the text files written by pgen.""" - self.parse_graminit_h(graminit_h) - self.parse_graminit_c(graminit_c) - self.finish_off() - - def parse_graminit_h(self, filename): - """Parse the .h file written by pgen. (Internal) - - This file is a sequence of #define statements defining the - nonterminals of the grammar as numbers. We build two tables - mapping the numbers to names and back. - - """ - try: - f = open(filename) - except OSError as err: - print("Can't open %s: %s" % (filename, err)) - return False - self.symbol2number = {} - self.number2symbol = {} - lineno = 0 - for line in f: - lineno += 1 - mo = re.match(r"^#define\s+(\w+)\s+(\d+)$", line) - if not mo and line.strip(): - print("%s(%s): can't parse %s" % (filename, lineno, - line.strip())) - else: - symbol, number = mo.groups() - number = int(number) - assert symbol not in self.symbol2number - assert number not in self.number2symbol - self.symbol2number[symbol] = number - self.number2symbol[number] = symbol - return True - - def parse_graminit_c(self, filename): - """Parse the .c file written by pgen. (Internal) - - The file looks as follows. The first two lines are always this: - - #include "pgenheaders.h" - #include "grammar.h" - - After that come four blocks: - - 1) one or more state definitions - 2) a table defining dfas - 3) a table defining labels - 4) a struct defining the grammar - - A state definition has the following form: - - one or more arc arrays, each of the form: - static arc arcs__[] = { - {, }, - ... - }; - - followed by a state array, of the form: - static state states_[] = { - {, arcs__}, - ... - }; - - """ - try: - f = open(filename) - except OSError as err: - print("Can't open %s: %s" % (filename, err)) - return False - # The code below essentially uses f's iterator-ness! - lineno = 0 - - # Expect the two #include lines - lineno, line = lineno+1, next(f) - assert line == '#include "pgenheaders.h"\n', (lineno, line) - lineno, line = lineno+1, next(f) - assert line == '#include "grammar.h"\n', (lineno, line) - - # Parse the state definitions - lineno, line = lineno+1, next(f) - allarcs = {} - states = [] - while line.startswith("static arc "): - while line.startswith("static arc "): - mo = re.match(r"static arc arcs_(\d+)_(\d+)\[(\d+)\] = {$", - line) - assert mo, (lineno, line) - n, m, k = list(map(int, mo.groups())) - arcs = [] - for _ in range(k): - lineno, line = lineno+1, next(f) - mo = re.match(r"\s+{(\d+), (\d+)},$", line) - assert mo, (lineno, line) - i, j = list(map(int, mo.groups())) - arcs.append((i, j)) - lineno, line = lineno+1, next(f) - assert line == "};\n", (lineno, line) - allarcs[(n, m)] = arcs - lineno, line = lineno+1, next(f) - mo = re.match(r"static state states_(\d+)\[(\d+)\] = {$", line) - assert mo, (lineno, line) - s, t = list(map(int, mo.groups())) - assert s == len(states), (lineno, line) - state = [] - for _ in range(t): - lineno, line = lineno+1, next(f) - mo = re.match(r"\s+{(\d+), arcs_(\d+)_(\d+)},$", line) - assert mo, (lineno, line) - k, n, m = list(map(int, mo.groups())) - arcs = allarcs[n, m] - assert k == len(arcs), (lineno, line) - state.append(arcs) - states.append(state) - lineno, line = lineno+1, next(f) - assert line == "};\n", (lineno, line) - lineno, line = lineno+1, next(f) - self.states = states - - # Parse the dfas - dfas = {} - mo = re.match(r"static dfa dfas\[(\d+)\] = {$", line) - assert mo, (lineno, line) - ndfas = int(mo.group(1)) - for i in range(ndfas): - lineno, line = lineno+1, next(f) - mo = re.match(r'\s+{(\d+), "(\w+)", (\d+), (\d+), states_(\d+),$', - line) - assert mo, (lineno, line) - symbol = mo.group(2) - number, x, y, z = list(map(int, mo.group(1, 3, 4, 5))) - assert self.symbol2number[symbol] == number, (lineno, line) - assert self.number2symbol[number] == symbol, (lineno, line) - assert x == 0, (lineno, line) - state = states[z] - assert y == len(state), (lineno, line) - lineno, line = lineno+1, next(f) - mo = re.match(r'\s+("(?:\\\d\d\d)*")},$', line) - assert mo, (lineno, line) - first = {} - rawbitset = eval(mo.group(1)) - for i, c in enumerate(rawbitset): - byte = ord(c) - for j in range(8): - if byte & (1<= os.path.getmtime(b) - - -def load_packaged_grammar(package, grammar_source): - """Normally, loads a pickled grammar by doing - pkgutil.get_data(package, pickled_grammar) - where *pickled_grammar* is computed from *grammar_source* by adding the - Python version and using a ``.pickle`` extension. - - However, if *grammar_source* is an extant file, load_grammar(grammar_source) - is called instead. This facilitates using a packaged grammar file when needed - but preserves load_grammar's automatic regeneration behavior when possible. - - """ - if os.path.isfile(grammar_source): - return load_grammar(grammar_source) - pickled_name = _generate_pickle_name(os.path.basename(grammar_source)) - data = pkgutil.get_data(package, pickled_name) - g = grammar.Grammar() - g.loads(data) - return g - - -def main(*args): - """Main program, when run as a script: produce grammar pickle files. - - Calls load_grammar for each argument, a path to a grammar text file. - """ - if not args: - args = sys.argv[1:] - logging.basicConfig(level=logging.INFO, stream=sys.stdout, - format='%(message)s') - for gt in args: - load_grammar(gt, save=True, force=True) - return True - -if __name__ == "__main__": - sys.exit(int(not main())) diff --git a/Lib/lib2to3/pgen2/grammar.py b/Lib/lib2to3/pgen2/grammar.py deleted file mode 100644 index 5d550aeb65e8dd..00000000000000 --- a/Lib/lib2to3/pgen2/grammar.py +++ /dev/null @@ -1,189 +0,0 @@ -# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""This module defines the data structures used to represent a grammar. - -These are a bit arcane because they are derived from the data -structures used by Python's 'pgen' parser generator. - -There's also a table here mapping operators to their names in the -token module; the Python tokenize module reports all operators as the -fallback token code OP, but the parser needs the actual token code. - -""" - -# Python imports -import pickle - -# Local imports -from . import token - - -class Grammar(object): - """Pgen parsing tables conversion class. - - Once initialized, this class supplies the grammar tables for the - parsing engine implemented by parse.py. The parsing engine - accesses the instance variables directly. The class here does not - provide initialization of the tables; several subclasses exist to - do this (see the conv and pgen modules). - - The load() method reads the tables from a pickle file, which is - much faster than the other ways offered by subclasses. The pickle - file is written by calling dump() (after loading the grammar - tables using a subclass). The report() method prints a readable - representation of the tables to stdout, for debugging. - - The instance variables are as follows: - - symbol2number -- a dict mapping symbol names to numbers. Symbol - numbers are always 256 or higher, to distinguish - them from token numbers, which are between 0 and - 255 (inclusive). - - number2symbol -- a dict mapping numbers to symbol names; - these two are each other's inverse. - - states -- a list of DFAs, where each DFA is a list of - states, each state is a list of arcs, and each - arc is a (i, j) pair where i is a label and j is - a state number. The DFA number is the index into - this list. (This name is slightly confusing.) - Final states are represented by a special arc of - the form (0, j) where j is its own state number. - - dfas -- a dict mapping symbol numbers to (DFA, first) - pairs, where DFA is an item from the states list - above, and first is a set of tokens that can - begin this grammar rule (represented by a dict - whose values are always 1). - - labels -- a list of (x, y) pairs where x is either a token - number or a symbol number, and y is either None - or a string; the strings are keywords. The label - number is the index in this list; label numbers - are used to mark state transitions (arcs) in the - DFAs. - - start -- the number of the grammar's start symbol. - - keywords -- a dict mapping keyword strings to arc labels. - - tokens -- a dict mapping token numbers to arc labels. - - """ - - def __init__(self): - self.symbol2number = {} - self.number2symbol = {} - self.states = [] - self.dfas = {} - self.labels = [(0, "EMPTY")] - self.keywords = {} - self.tokens = {} - self.symbol2label = {} - self.start = 256 - - def dump(self, filename): - """Dump the grammar tables to a pickle file.""" - with open(filename, "wb") as f: - pickle.dump(self.__dict__, f, pickle.HIGHEST_PROTOCOL) - - def load(self, filename): - """Load the grammar tables from a pickle file.""" - with open(filename, "rb") as f: - d = pickle.load(f) - self.__dict__.update(d) - - def loads(self, pkl): - """Load the grammar tables from a pickle bytes object.""" - self.__dict__.update(pickle.loads(pkl)) - - def copy(self): - """ - Copy the grammar. - """ - new = self.__class__() - for dict_attr in ("symbol2number", "number2symbol", "dfas", "keywords", - "tokens", "symbol2label"): - setattr(new, dict_attr, getattr(self, dict_attr).copy()) - new.labels = self.labels[:] - new.states = self.states[:] - new.start = self.start - return new - - def report(self): - """Dump the grammar tables to standard output, for debugging.""" - from pprint import pprint - print("s2n") - pprint(self.symbol2number) - print("n2s") - pprint(self.number2symbol) - print("states") - pprint(self.states) - print("dfas") - pprint(self.dfas) - print("labels") - pprint(self.labels) - print("start", self.start) - - -# Map from operator to number (since tokenize doesn't do this) - -opmap_raw = """ -( LPAR -) RPAR -[ LSQB -] RSQB -: COLON -, COMMA -; SEMI -+ PLUS -- MINUS -* STAR -/ SLASH -| VBAR -& AMPER -< LESS -> GREATER -= EQUAL -. DOT -% PERCENT -` BACKQUOTE -{ LBRACE -} RBRACE -@ AT -@= ATEQUAL -== EQEQUAL -!= NOTEQUAL -<> NOTEQUAL -<= LESSEQUAL ->= GREATEREQUAL -~ TILDE -^ CIRCUMFLEX -<< LEFTSHIFT ->> RIGHTSHIFT -** DOUBLESTAR -+= PLUSEQUAL --= MINEQUAL -*= STAREQUAL -/= SLASHEQUAL -%= PERCENTEQUAL -&= AMPEREQUAL -|= VBAREQUAL -^= CIRCUMFLEXEQUAL -<<= LEFTSHIFTEQUAL ->>= RIGHTSHIFTEQUAL -**= DOUBLESTAREQUAL -// DOUBLESLASH -//= DOUBLESLASHEQUAL --> RARROW -:= COLONEQUAL -""" - -opmap = {} -for line in opmap_raw.splitlines(): - if line: - op, name = line.split() - opmap[op] = getattr(token, name) -del line, op, name diff --git a/Lib/lib2to3/pgen2/literals.py b/Lib/lib2to3/pgen2/literals.py deleted file mode 100644 index b9b63e6e5572c1..00000000000000 --- a/Lib/lib2to3/pgen2/literals.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Safely evaluate Python string literals without using eval().""" - -import re - -simple_escapes = {"a": "\a", - "b": "\b", - "f": "\f", - "n": "\n", - "r": "\r", - "t": "\t", - "v": "\v", - "'": "'", - '"': '"', - "\\": "\\"} - -def escape(m): - all, tail = m.group(0, 1) - assert all.startswith("\\") - esc = simple_escapes.get(tail) - if esc is not None: - return esc - if tail.startswith("x"): - hexes = tail[1:] - if len(hexes) < 2: - raise ValueError("invalid hex string escape ('\\%s')" % tail) - try: - i = int(hexes, 16) - except ValueError: - raise ValueError("invalid hex string escape ('\\%s')" % tail) from None - else: - try: - i = int(tail, 8) - except ValueError: - raise ValueError("invalid octal string escape ('\\%s')" % tail) from None - return chr(i) - -def evalString(s): - assert s.startswith("'") or s.startswith('"'), repr(s[:1]) - q = s[0] - if s[:3] == q*3: - q = q*3 - assert s.endswith(q), repr(s[-len(q):]) - assert len(s) >= 2*len(q) - s = s[len(q):-len(q)] - return re.sub(r"\\(\'|\"|\\|[abfnrtv]|x.{0,2}|[0-7]{1,3})", escape, s) - -def test(): - for i in range(256): - c = chr(i) - s = repr(c) - e = evalString(s) - if e != c: - print(i, c, s, e) - - -if __name__ == "__main__": - test() diff --git a/Lib/lib2to3/pgen2/parse.py b/Lib/lib2to3/pgen2/parse.py deleted file mode 100644 index cf3fcf7e99fd11..00000000000000 --- a/Lib/lib2to3/pgen2/parse.py +++ /dev/null @@ -1,204 +0,0 @@ -# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Parser engine for the grammar tables generated by pgen. - -The grammar table must be loaded first. - -See Parser/parser.c in the Python distribution for additional info on -how this parsing engine works. - -""" - -# Local imports -from . import token - -class ParseError(Exception): - """Exception to signal the parser is stuck.""" - - def __init__(self, msg, type, value, context): - Exception.__init__(self, "%s: type=%r, value=%r, context=%r" % - (msg, type, value, context)) - self.msg = msg - self.type = type - self.value = value - self.context = context - - def __reduce__(self): - return type(self), (self.msg, self.type, self.value, self.context) - -class Parser(object): - """Parser engine. - - The proper usage sequence is: - - p = Parser(grammar, [converter]) # create instance - p.setup([start]) # prepare for parsing - : - if p.addtoken(...): # parse a token; may raise ParseError - break - root = p.rootnode # root of abstract syntax tree - - A Parser instance may be reused by calling setup() repeatedly. - - A Parser instance contains state pertaining to the current token - sequence, and should not be used concurrently by different threads - to parse separate token sequences. - - See driver.py for how to get input tokens by tokenizing a file or - string. - - Parsing is complete when addtoken() returns True; the root of the - abstract syntax tree can then be retrieved from the rootnode - instance variable. When a syntax error occurs, addtoken() raises - the ParseError exception. There is no error recovery; the parser - cannot be used after a syntax error was reported (but it can be - reinitialized by calling setup()). - - """ - - def __init__(self, grammar, convert=None): - """Constructor. - - The grammar argument is a grammar.Grammar instance; see the - grammar module for more information. - - The parser is not ready yet for parsing; you must call the - setup() method to get it started. - - The optional convert argument is a function mapping concrete - syntax tree nodes to abstract syntax tree nodes. If not - given, no conversion is done and the syntax tree produced is - the concrete syntax tree. If given, it must be a function of - two arguments, the first being the grammar (a grammar.Grammar - instance), and the second being the concrete syntax tree node - to be converted. The syntax tree is converted from the bottom - up. - - A concrete syntax tree node is a (type, value, context, nodes) - tuple, where type is the node type (a token or symbol number), - value is None for symbols and a string for tokens, context is - None or an opaque value used for error reporting (typically a - (lineno, offset) pair), and nodes is a list of children for - symbols, and None for tokens. - - An abstract syntax tree node may be anything; this is entirely - up to the converter function. - - """ - self.grammar = grammar - self.convert = convert or (lambda grammar, node: node) - - def setup(self, start=None): - """Prepare for parsing. - - This *must* be called before starting to parse. - - The optional argument is an alternative start symbol; it - defaults to the grammar's start symbol. - - You can use a Parser instance to parse any number of programs; - each time you call setup() the parser is reset to an initial - state determined by the (implicit or explicit) start symbol. - - """ - if start is None: - start = self.grammar.start - # Each stack entry is a tuple: (dfa, state, node). - # A node is a tuple: (type, value, context, children), - # where children is a list of nodes or None, and context may be None. - newnode = (start, None, None, []) - stackentry = (self.grammar.dfas[start], 0, newnode) - self.stack = [stackentry] - self.rootnode = None - self.used_names = set() # Aliased to self.rootnode.used_names in pop() - - def addtoken(self, type, value, context): - """Add a token; return True iff this is the end of the program.""" - # Map from token to label - ilabel = self.classify(type, value, context) - # Loop until the token is shifted; may raise exceptions - while True: - dfa, state, node = self.stack[-1] - states, first = dfa - arcs = states[state] - # Look for a state with this label - for i, newstate in arcs: - t, v = self.grammar.labels[i] - if ilabel == i: - # Look it up in the list of labels - assert t < 256 - # Shift a token; we're done with it - self.shift(type, value, newstate, context) - # Pop while we are in an accept-only state - state = newstate - while states[state] == [(0, state)]: - self.pop() - if not self.stack: - # Done parsing! - return True - dfa, state, node = self.stack[-1] - states, first = dfa - # Done with this token - return False - elif t >= 256: - # See if it's a symbol and if we're in its first set - itsdfa = self.grammar.dfas[t] - itsstates, itsfirst = itsdfa - if ilabel in itsfirst: - # Push a symbol - self.push(t, self.grammar.dfas[t], newstate, context) - break # To continue the outer while loop - else: - if (0, state) in arcs: - # An accepting state, pop it and try something else - self.pop() - if not self.stack: - # Done parsing, but another token is input - raise ParseError("too much input", - type, value, context) - else: - # No success finding a transition - raise ParseError("bad input", type, value, context) - - def classify(self, type, value, context): - """Turn a token into a label. (Internal)""" - if type == token.NAME: - # Keep a listing of all used names - self.used_names.add(value) - # Check for reserved words - ilabel = self.grammar.keywords.get(value) - if ilabel is not None: - return ilabel - ilabel = self.grammar.tokens.get(type) - if ilabel is None: - raise ParseError("bad token", type, value, context) - return ilabel - - def shift(self, type, value, newstate, context): - """Shift a token. (Internal)""" - dfa, state, node = self.stack[-1] - newnode = (type, value, context, None) - newnode = self.convert(self.grammar, newnode) - if newnode is not None: - node[-1].append(newnode) - self.stack[-1] = (dfa, newstate, node) - - def push(self, type, newdfa, newstate, context): - """Push a nonterminal. (Internal)""" - dfa, state, node = self.stack[-1] - newnode = (type, None, context, []) - self.stack[-1] = (dfa, newstate, node) - self.stack.append((newdfa, 0, newnode)) - - def pop(self): - """Pop a nonterminal. (Internal)""" - popdfa, popstate, popnode = self.stack.pop() - newnode = self.convert(self.grammar, popnode) - if newnode is not None: - if self.stack: - dfa, state, node = self.stack[-1] - node[-1].append(newnode) - else: - self.rootnode = newnode - self.rootnode.used_names = self.used_names diff --git a/Lib/lib2to3/pgen2/pgen.py b/Lib/lib2to3/pgen2/pgen.py deleted file mode 100644 index 7abd5cef1c36bb..00000000000000 --- a/Lib/lib2to3/pgen2/pgen.py +++ /dev/null @@ -1,386 +0,0 @@ -# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -# Pgen imports -from . import grammar, token, tokenize - -class PgenGrammar(grammar.Grammar): - pass - -class ParserGenerator(object): - - def __init__(self, filename, stream=None): - close_stream = None - if stream is None: - stream = open(filename, encoding="utf-8") - close_stream = stream.close - self.filename = filename - self.stream = stream - self.generator = tokenize.generate_tokens(stream.readline) - self.gettoken() # Initialize lookahead - self.dfas, self.startsymbol = self.parse() - if close_stream is not None: - close_stream() - self.first = {} # map from symbol name to set of tokens - self.addfirstsets() - - def make_grammar(self): - c = PgenGrammar() - names = list(self.dfas.keys()) - names.sort() - names.remove(self.startsymbol) - names.insert(0, self.startsymbol) - for name in names: - i = 256 + len(c.symbol2number) - c.symbol2number[name] = i - c.number2symbol[i] = name - for name in names: - dfa = self.dfas[name] - states = [] - for state in dfa: - arcs = [] - for label, next in sorted(state.arcs.items()): - arcs.append((self.make_label(c, label), dfa.index(next))) - if state.isfinal: - arcs.append((0, dfa.index(state))) - states.append(arcs) - c.states.append(states) - c.dfas[c.symbol2number[name]] = (states, self.make_first(c, name)) - c.start = c.symbol2number[self.startsymbol] - return c - - def make_first(self, c, name): - rawfirst = self.first[name] - first = {} - for label in sorted(rawfirst): - ilabel = self.make_label(c, label) - ##assert ilabel not in first # XXX failed on <> ... != - first[ilabel] = 1 - return first - - def make_label(self, c, label): - # XXX Maybe this should be a method on a subclass of converter? - ilabel = len(c.labels) - if label[0].isalpha(): - # Either a symbol name or a named token - if label in c.symbol2number: - # A symbol name (a non-terminal) - if label in c.symbol2label: - return c.symbol2label[label] - else: - c.labels.append((c.symbol2number[label], None)) - c.symbol2label[label] = ilabel - return ilabel - else: - # A named token (NAME, NUMBER, STRING) - itoken = getattr(token, label, None) - assert isinstance(itoken, int), label - assert itoken in token.tok_name, label - if itoken in c.tokens: - return c.tokens[itoken] - else: - c.labels.append((itoken, None)) - c.tokens[itoken] = ilabel - return ilabel - else: - # Either a keyword or an operator - assert label[0] in ('"', "'"), label - value = eval(label) - if value[0].isalpha(): - # A keyword - if value in c.keywords: - return c.keywords[value] - else: - c.labels.append((token.NAME, value)) - c.keywords[value] = ilabel - return ilabel - else: - # An operator (any non-numeric token) - itoken = grammar.opmap[value] # Fails if unknown token - if itoken in c.tokens: - return c.tokens[itoken] - else: - c.labels.append((itoken, None)) - c.tokens[itoken] = ilabel - return ilabel - - def addfirstsets(self): - names = list(self.dfas.keys()) - names.sort() - for name in names: - if name not in self.first: - self.calcfirst(name) - #print name, self.first[name].keys() - - def calcfirst(self, name): - dfa = self.dfas[name] - self.first[name] = None # dummy to detect left recursion - state = dfa[0] - totalset = {} - overlapcheck = {} - for label, next in state.arcs.items(): - if label in self.dfas: - if label in self.first: - fset = self.first[label] - if fset is None: - raise ValueError("recursion for rule %r" % name) - else: - self.calcfirst(label) - fset = self.first[label] - totalset.update(fset) - overlapcheck[label] = fset - else: - totalset[label] = 1 - overlapcheck[label] = {label: 1} - inverse = {} - for label, itsfirst in overlapcheck.items(): - for symbol in itsfirst: - if symbol in inverse: - raise ValueError("rule %s is ambiguous; %s is in the" - " first sets of %s as well as %s" % - (name, symbol, label, inverse[symbol])) - inverse[symbol] = label - self.first[name] = totalset - - def parse(self): - dfas = {} - startsymbol = None - # MSTART: (NEWLINE | RULE)* ENDMARKER - while self.type != token.ENDMARKER: - while self.type == token.NEWLINE: - self.gettoken() - # RULE: NAME ':' RHS NEWLINE - name = self.expect(token.NAME) - self.expect(token.OP, ":") - a, z = self.parse_rhs() - self.expect(token.NEWLINE) - #self.dump_nfa(name, a, z) - dfa = self.make_dfa(a, z) - #self.dump_dfa(name, dfa) - oldlen = len(dfa) - self.simplify_dfa(dfa) - newlen = len(dfa) - dfas[name] = dfa - #print name, oldlen, newlen - if startsymbol is None: - startsymbol = name - return dfas, startsymbol - - def make_dfa(self, start, finish): - # To turn an NFA into a DFA, we define the states of the DFA - # to correspond to *sets* of states of the NFA. Then do some - # state reduction. Let's represent sets as dicts with 1 for - # values. - assert isinstance(start, NFAState) - assert isinstance(finish, NFAState) - def closure(state): - base = {} - addclosure(state, base) - return base - def addclosure(state, base): - assert isinstance(state, NFAState) - if state in base: - return - base[state] = 1 - for label, next in state.arcs: - if label is None: - addclosure(next, base) - states = [DFAState(closure(start), finish)] - for state in states: # NB states grows while we're iterating - arcs = {} - for nfastate in state.nfaset: - for label, next in nfastate.arcs: - if label is not None: - addclosure(next, arcs.setdefault(label, {})) - for label, nfaset in sorted(arcs.items()): - for st in states: - if st.nfaset == nfaset: - break - else: - st = DFAState(nfaset, finish) - states.append(st) - state.addarc(st, label) - return states # List of DFAState instances; first one is start - - def dump_nfa(self, name, start, finish): - print("Dump of NFA for", name) - todo = [start] - for i, state in enumerate(todo): - print(" State", i, state is finish and "(final)" or "") - for label, next in state.arcs: - if next in todo: - j = todo.index(next) - else: - j = len(todo) - todo.append(next) - if label is None: - print(" -> %d" % j) - else: - print(" %s -> %d" % (label, j)) - - def dump_dfa(self, name, dfa): - print("Dump of DFA for", name) - for i, state in enumerate(dfa): - print(" State", i, state.isfinal and "(final)" or "") - for label, next in sorted(state.arcs.items()): - print(" %s -> %d" % (label, dfa.index(next))) - - def simplify_dfa(self, dfa): - # This is not theoretically optimal, but works well enough. - # Algorithm: repeatedly look for two states that have the same - # set of arcs (same labels pointing to the same nodes) and - # unify them, until things stop changing. - - # dfa is a list of DFAState instances - changes = True - while changes: - changes = False - for i, state_i in enumerate(dfa): - for j in range(i+1, len(dfa)): - state_j = dfa[j] - if state_i == state_j: - #print " unify", i, j - del dfa[j] - for state in dfa: - state.unifystate(state_j, state_i) - changes = True - break - - def parse_rhs(self): - # RHS: ALT ('|' ALT)* - a, z = self.parse_alt() - if self.value != "|": - return a, z - else: - aa = NFAState() - zz = NFAState() - aa.addarc(a) - z.addarc(zz) - while self.value == "|": - self.gettoken() - a, z = self.parse_alt() - aa.addarc(a) - z.addarc(zz) - return aa, zz - - def parse_alt(self): - # ALT: ITEM+ - a, b = self.parse_item() - while (self.value in ("(", "[") or - self.type in (token.NAME, token.STRING)): - c, d = self.parse_item() - b.addarc(c) - b = d - return a, b - - def parse_item(self): - # ITEM: '[' RHS ']' | ATOM ['+' | '*'] - if self.value == "[": - self.gettoken() - a, z = self.parse_rhs() - self.expect(token.OP, "]") - a.addarc(z) - return a, z - else: - a, z = self.parse_atom() - value = self.value - if value not in ("+", "*"): - return a, z - self.gettoken() - z.addarc(a) - if value == "+": - return a, z - else: - return a, a - - def parse_atom(self): - # ATOM: '(' RHS ')' | NAME | STRING - if self.value == "(": - self.gettoken() - a, z = self.parse_rhs() - self.expect(token.OP, ")") - return a, z - elif self.type in (token.NAME, token.STRING): - a = NFAState() - z = NFAState() - a.addarc(z, self.value) - self.gettoken() - return a, z - else: - self.raise_error("expected (...) or NAME or STRING, got %s/%s", - self.type, self.value) - - def expect(self, type, value=None): - if self.type != type or (value is not None and self.value != value): - self.raise_error("expected %s/%s, got %s/%s", - type, value, self.type, self.value) - value = self.value - self.gettoken() - return value - - def gettoken(self): - tup = next(self.generator) - while tup[0] in (tokenize.COMMENT, tokenize.NL): - tup = next(self.generator) - self.type, self.value, self.begin, self.end, self.line = tup - #print token.tok_name[self.type], repr(self.value) - - def raise_error(self, msg, *args): - if args: - try: - msg = msg % args - except: - msg = " ".join([msg] + list(map(str, args))) - raise SyntaxError(msg, (self.filename, self.end[0], - self.end[1], self.line)) - -class NFAState(object): - - def __init__(self): - self.arcs = [] # list of (label, NFAState) pairs - - def addarc(self, next, label=None): - assert label is None or isinstance(label, str) - assert isinstance(next, NFAState) - self.arcs.append((label, next)) - -class DFAState(object): - - def __init__(self, nfaset, final): - assert isinstance(nfaset, dict) - assert isinstance(next(iter(nfaset)), NFAState) - assert isinstance(final, NFAState) - self.nfaset = nfaset - self.isfinal = final in nfaset - self.arcs = {} # map from label to DFAState - - def addarc(self, next, label): - assert isinstance(label, str) - assert label not in self.arcs - assert isinstance(next, DFAState) - self.arcs[label] = next - - def unifystate(self, old, new): - for label, next in self.arcs.items(): - if next is old: - self.arcs[label] = new - - def __eq__(self, other): - # Equality test -- ignore the nfaset instance variable - assert isinstance(other, DFAState) - if self.isfinal != other.isfinal: - return False - # Can't just return self.arcs == other.arcs, because that - # would invoke this method recursively, with cycles... - if len(self.arcs) != len(other.arcs): - return False - for label, next in self.arcs.items(): - if next is not other.arcs.get(label): - return False - return True - - __hash__ = None # For Py3 compatibility. - -def generate_grammar(filename="Grammar.txt"): - p = ParserGenerator(filename) - return p.make_grammar() diff --git a/Lib/lib2to3/pgen2/token.py b/Lib/lib2to3/pgen2/token.py deleted file mode 100755 index 2a55138e48237b..00000000000000 --- a/Lib/lib2to3/pgen2/token.py +++ /dev/null @@ -1,86 +0,0 @@ -#! /usr/bin/env python3 - -"""Token constants (from "token.h").""" - -# Taken from Python (r53757) and modified to include some tokens -# originally monkeypatched in by pgen2.tokenize - -#--start constants-- -ENDMARKER = 0 -NAME = 1 -NUMBER = 2 -STRING = 3 -NEWLINE = 4 -INDENT = 5 -DEDENT = 6 -LPAR = 7 -RPAR = 8 -LSQB = 9 -RSQB = 10 -COLON = 11 -COMMA = 12 -SEMI = 13 -PLUS = 14 -MINUS = 15 -STAR = 16 -SLASH = 17 -VBAR = 18 -AMPER = 19 -LESS = 20 -GREATER = 21 -EQUAL = 22 -DOT = 23 -PERCENT = 24 -BACKQUOTE = 25 -LBRACE = 26 -RBRACE = 27 -EQEQUAL = 28 -NOTEQUAL = 29 -LESSEQUAL = 30 -GREATEREQUAL = 31 -TILDE = 32 -CIRCUMFLEX = 33 -LEFTSHIFT = 34 -RIGHTSHIFT = 35 -DOUBLESTAR = 36 -PLUSEQUAL = 37 -MINEQUAL = 38 -STAREQUAL = 39 -SLASHEQUAL = 40 -PERCENTEQUAL = 41 -AMPEREQUAL = 42 -VBAREQUAL = 43 -CIRCUMFLEXEQUAL = 44 -LEFTSHIFTEQUAL = 45 -RIGHTSHIFTEQUAL = 46 -DOUBLESTAREQUAL = 47 -DOUBLESLASH = 48 -DOUBLESLASHEQUAL = 49 -AT = 50 -ATEQUAL = 51 -OP = 52 -COMMENT = 53 -NL = 54 -RARROW = 55 -AWAIT = 56 -ASYNC = 57 -ERRORTOKEN = 58 -COLONEQUAL = 59 -N_TOKENS = 60 -NT_OFFSET = 256 -#--end constants-- - -tok_name = {} -for _name, _value in list(globals().items()): - if isinstance(_value, int): - tok_name[_value] = _name - - -def ISTERMINAL(x): - return x < NT_OFFSET - -def ISNONTERMINAL(x): - return x >= NT_OFFSET - -def ISEOF(x): - return x == ENDMARKER diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py deleted file mode 100644 index 099dfa7798afd4..00000000000000 --- a/Lib/lib2to3/pgen2/tokenize.py +++ /dev/null @@ -1,564 +0,0 @@ -# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006 Python Software Foundation. -# All rights reserved. - -"""Tokenization help for Python programs. - -generate_tokens(readline) is a generator that breaks a stream of -text into Python tokens. It accepts a readline-like method which is called -repeatedly to get the next line of input (or "" for EOF). It generates -5-tuples with these members: - - the token type (see token.py) - the token (a string) - the starting (row, column) indices of the token (a 2-tuple of ints) - the ending (row, column) indices of the token (a 2-tuple of ints) - the original line (string) - -It is designed to match the working of the Python tokenizer exactly, except -that it produces COMMENT tokens for comments and gives type OP for all -operators - -Older entry points - tokenize_loop(readline, tokeneater) - tokenize(readline, tokeneater=printtoken) -are the same, except instead of generating tokens, tokeneater is a callback -function to which the 5 fields described above are passed as 5 arguments, -each time a new token is found.""" - -__author__ = 'Ka-Ping Yee ' -__credits__ = \ - 'GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro' - -import string, re -from codecs import BOM_UTF8, lookup -from lib2to3.pgen2.token import * - -from . import token -__all__ = [x for x in dir(token) if x[0] != '_'] + ["tokenize", - "generate_tokens", "untokenize"] -del token - -try: - bytes -except NameError: - # Support bytes type in Python <= 2.5, so 2to3 turns itself into - # valid Python 3 code. - bytes = str - -def group(*choices): return '(' + '|'.join(choices) + ')' -def any(*choices): return group(*choices) + '*' -def maybe(*choices): return group(*choices) + '?' -def _combinations(*l): - return set( - x + y for x in l for y in l + ("",) if x.casefold() != y.casefold() - ) - -Whitespace = r'[ \f\t]*' -Comment = r'#[^\r\n]*' -Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) -Name = r'\w+' - -Binnumber = r'0[bB]_?[01]+(?:_[01]+)*' -Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?' -Octnumber = r'0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?' -Decnumber = group(r'[1-9]\d*(?:_\d+)*[lL]?', '0[lL]?') -Intnumber = group(Binnumber, Hexnumber, Octnumber, Decnumber) -Exponent = r'[eE][-+]?\d+(?:_\d+)*' -Pointfloat = group(r'\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?', r'\.\d+(?:_\d+)*') + maybe(Exponent) -Expfloat = r'\d+(?:_\d+)*' + Exponent -Floatnumber = group(Pointfloat, Expfloat) -Imagnumber = group(r'\d+(?:_\d+)*[jJ]', Floatnumber + r'[jJ]') -Number = group(Imagnumber, Floatnumber, Intnumber) - -# Tail end of ' string. -Single = r"[^'\\]*(?:\\.[^'\\]*)*'" -# Tail end of " string. -Double = r'[^"\\]*(?:\\.[^"\\]*)*"' -# Tail end of ''' string. -Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''" -# Tail end of """ string. -Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""' -_litprefix = r"(?:[uUrRbBfF]|[rR][fFbB]|[fFbBuU][rR])?" -Triple = group(_litprefix + "'''", _litprefix + '"""') -# Single-line ' or " string. -String = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'", - _litprefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"') - -# Because of leftmost-then-longest match semantics, be sure to put the -# longest operators first (e.g., if = came before ==, == would get -# recognized as two instances of =). -Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=", - r"//=?", r"->", - r"[+\-*/%&@|^=<>]=?", - r"~") - -Bracket = '[][(){}]' -Special = group(r'\r?\n', r':=', r'[:;.,`@]') -Funny = group(Operator, Bracket, Special) - -PlainToken = group(Number, Funny, String, Name) -Token = Ignore + PlainToken - -# First (or only) line of ' or " string. -ContStr = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" + - group("'", r'\\\r?\n'), - _litprefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' + - group('"', r'\\\r?\n')) -PseudoExtras = group(r'\\\r?\n', Comment, Triple) -PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) - -tokenprog, pseudoprog, single3prog, double3prog = map( - re.compile, (Token, PseudoToken, Single3, Double3)) - -_strprefixes = ( - _combinations('r', 'R', 'f', 'F') | - _combinations('r', 'R', 'b', 'B') | - {'u', 'U', 'ur', 'uR', 'Ur', 'UR'} -) - -endprogs = {"'": re.compile(Single), '"': re.compile(Double), - "'''": single3prog, '"""': double3prog, - **{f"{prefix}'''": single3prog for prefix in _strprefixes}, - **{f'{prefix}"""': double3prog for prefix in _strprefixes}, - **{prefix: None for prefix in _strprefixes}} - -triple_quoted = ( - {"'''", '"""'} | - {f"{prefix}'''" for prefix in _strprefixes} | - {f'{prefix}"""' for prefix in _strprefixes} -) -single_quoted = ( - {"'", '"'} | - {f"{prefix}'" for prefix in _strprefixes} | - {f'{prefix}"' for prefix in _strprefixes} -) - -tabsize = 8 - -class TokenError(Exception): pass - -class StopTokenizing(Exception): pass - -def printtoken(type, token, xxx_todo_changeme, xxx_todo_changeme1, line): # for testing - (srow, scol) = xxx_todo_changeme - (erow, ecol) = xxx_todo_changeme1 - print("%d,%d-%d,%d:\t%s\t%s" % \ - (srow, scol, erow, ecol, tok_name[type], repr(token))) - -def tokenize(readline, tokeneater=printtoken): - """ - The tokenize() function accepts two parameters: one representing the - input stream, and one providing an output mechanism for tokenize(). - - The first parameter, readline, must be a callable object which provides - the same interface as the readline() method of built-in file objects. - Each call to the function should return one line of input as a string. - - The second parameter, tokeneater, must also be a callable object. It is - called once for each token, with five arguments, corresponding to the - tuples generated by generate_tokens(). - """ - try: - tokenize_loop(readline, tokeneater) - except StopTokenizing: - pass - -# backwards compatible interface -def tokenize_loop(readline, tokeneater): - for token_info in generate_tokens(readline): - tokeneater(*token_info) - -class Untokenizer: - - def __init__(self): - self.tokens = [] - self.prev_row = 1 - self.prev_col = 0 - - def add_whitespace(self, start): - row, col = start - assert row <= self.prev_row - col_offset = col - self.prev_col - if col_offset: - self.tokens.append(" " * col_offset) - - def untokenize(self, iterable): - for t in iterable: - if len(t) == 2: - self.compat(t, iterable) - break - tok_type, token, start, end, line = t - self.add_whitespace(start) - self.tokens.append(token) - self.prev_row, self.prev_col = end - if tok_type in (NEWLINE, NL): - self.prev_row += 1 - self.prev_col = 0 - return "".join(self.tokens) - - def compat(self, token, iterable): - startline = False - indents = [] - toks_append = self.tokens.append - toknum, tokval = token - if toknum in (NAME, NUMBER): - tokval += ' ' - if toknum in (NEWLINE, NL): - startline = True - for tok in iterable: - toknum, tokval = tok[:2] - - if toknum in (NAME, NUMBER, ASYNC, AWAIT): - tokval += ' ' - - if toknum == INDENT: - indents.append(tokval) - continue - elif toknum == DEDENT: - indents.pop() - continue - elif toknum in (NEWLINE, NL): - startline = True - elif startline and indents: - toks_append(indents[-1]) - startline = False - toks_append(tokval) - -cookie_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII) -blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII) - -def _get_normal_name(orig_enc): - """Imitates get_normal_name in tokenizer.c.""" - # Only care about the first 12 characters. - enc = orig_enc[:12].lower().replace("_", "-") - if enc == "utf-8" or enc.startswith("utf-8-"): - return "utf-8" - if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \ - enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")): - return "iso-8859-1" - return orig_enc - -def detect_encoding(readline): - """ - The detect_encoding() function is used to detect the encoding that should - be used to decode a Python source file. It requires one argument, readline, - in the same way as the tokenize() generator. - - It will call readline a maximum of twice, and return the encoding used - (as a string) and a list of any lines (left as bytes) it has read - in. - - It detects the encoding from the presence of a utf-8 bom or an encoding - cookie as specified in pep-0263. If both a bom and a cookie are present, but - disagree, a SyntaxError will be raised. If the encoding cookie is an invalid - charset, raise a SyntaxError. Note that if a utf-8 bom is found, - 'utf-8-sig' is returned. - - If no encoding is specified, then the default of 'utf-8' will be returned. - """ - bom_found = False - encoding = None - default = 'utf-8' - def read_or_stop(): - try: - return readline() - except StopIteration: - return bytes() - - def find_cookie(line): - try: - line_string = line.decode('ascii') - except UnicodeDecodeError: - return None - match = cookie_re.match(line_string) - if not match: - return None - encoding = _get_normal_name(match.group(1)) - try: - codec = lookup(encoding) - except LookupError: - # This behaviour mimics the Python interpreter - raise SyntaxError("unknown encoding: " + encoding) - - if bom_found: - if codec.name != 'utf-8': - # This behaviour mimics the Python interpreter - raise SyntaxError('encoding problem: utf-8') - encoding += '-sig' - return encoding - - first = read_or_stop() - if first.startswith(BOM_UTF8): - bom_found = True - first = first[3:] - default = 'utf-8-sig' - if not first: - return default, [] - - encoding = find_cookie(first) - if encoding: - return encoding, [first] - if not blank_re.match(first): - return default, [first] - - second = read_or_stop() - if not second: - return default, [first] - - encoding = find_cookie(second) - if encoding: - return encoding, [first, second] - - return default, [first, second] - -def untokenize(iterable): - """Transform tokens back into Python source code. - - Each element returned by the iterable must be a token sequence - with at least two elements, a token number and token value. If - only two tokens are passed, the resulting output is poor. - - Round-trip invariant for full input: - Untokenized source will match input source exactly - - Round-trip invariant for limited input: - # Output text will tokenize the back to the input - t1 = [tok[:2] for tok in generate_tokens(f.readline)] - newcode = untokenize(t1) - readline = iter(newcode.splitlines(1)).next - t2 = [tok[:2] for tokin generate_tokens(readline)] - assert t1 == t2 - """ - ut = Untokenizer() - return ut.untokenize(iterable) - -def generate_tokens(readline): - """ - The generate_tokens() generator requires one argument, readline, which - must be a callable object which provides the same interface as the - readline() method of built-in file objects. Each call to the function - should return one line of input as a string. Alternately, readline - can be a callable function terminating with StopIteration: - readline = open(myfile).next # Example of alternate readline - - The generator produces 5-tuples with these members: the token type; the - token string; a 2-tuple (srow, scol) of ints specifying the row and - column where the token begins in the source; a 2-tuple (erow, ecol) of - ints specifying the row and column where the token ends in the source; - and the line on which the token was found. The line passed is the - physical line. - """ - lnum = parenlev = continued = 0 - contstr, needcont = '', 0 - contline = None - indents = [0] - - # 'stashed' and 'async_*' are used for async/await parsing - stashed = None - async_def = False - async_def_indent = 0 - async_def_nl = False - - while 1: # loop over lines in stream - try: - line = readline() - except StopIteration: - line = '' - lnum = lnum + 1 - pos, max = 0, len(line) - - if contstr: # continued string - if not line: - raise TokenError("EOF in multi-line string", strstart) - endmatch = endprog.match(line) - if endmatch: - pos = end = endmatch.end(0) - yield (STRING, contstr + line[:end], - strstart, (lnum, end), contline + line) - contstr, needcont = '', 0 - contline = None - elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n': - yield (ERRORTOKEN, contstr + line, - strstart, (lnum, len(line)), contline) - contstr = '' - contline = None - continue - else: - contstr = contstr + line - contline = contline + line - continue - - elif parenlev == 0 and not continued: # new statement - if not line: break - column = 0 - while pos < max: # measure leading whitespace - if line[pos] == ' ': column = column + 1 - elif line[pos] == '\t': column = (column//tabsize + 1)*tabsize - elif line[pos] == '\f': column = 0 - else: break - pos = pos + 1 - if pos == max: break - - if stashed: - yield stashed - stashed = None - - if line[pos] in '#\r\n': # skip comments or blank lines - if line[pos] == '#': - comment_token = line[pos:].rstrip('\r\n') - nl_pos = pos + len(comment_token) - yield (COMMENT, comment_token, - (lnum, pos), (lnum, pos + len(comment_token)), line) - yield (NL, line[nl_pos:], - (lnum, nl_pos), (lnum, len(line)), line) - else: - yield ((NL, COMMENT)[line[pos] == '#'], line[pos:], - (lnum, pos), (lnum, len(line)), line) - continue - - if column > indents[-1]: # count indents or dedents - indents.append(column) - yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line) - while column < indents[-1]: - if column not in indents: - raise IndentationError( - "unindent does not match any outer indentation level", - ("", lnum, pos, line)) - indents = indents[:-1] - - if async_def and async_def_indent >= indents[-1]: - async_def = False - async_def_nl = False - async_def_indent = 0 - - yield (DEDENT, '', (lnum, pos), (lnum, pos), line) - - if async_def and async_def_nl and async_def_indent >= indents[-1]: - async_def = False - async_def_nl = False - async_def_indent = 0 - - else: # continued statement - if not line: - raise TokenError("EOF in multi-line statement", (lnum, 0)) - continued = 0 - - while pos < max: - pseudomatch = pseudoprog.match(line, pos) - if pseudomatch: # scan for tokens - start, end = pseudomatch.span(1) - spos, epos, pos = (lnum, start), (lnum, end), end - token, initial = line[start:end], line[start] - - if initial in string.digits or \ - (initial == '.' and token != '.'): # ordinary number - yield (NUMBER, token, spos, epos, line) - elif initial in '\r\n': - newline = NEWLINE - if parenlev > 0: - newline = NL - elif async_def: - async_def_nl = True - if stashed: - yield stashed - stashed = None - yield (newline, token, spos, epos, line) - - elif initial == '#': - assert not token.endswith("\n") - if stashed: - yield stashed - stashed = None - yield (COMMENT, token, spos, epos, line) - elif token in triple_quoted: - endprog = endprogs[token] - endmatch = endprog.match(line, pos) - if endmatch: # all on one line - pos = endmatch.end(0) - token = line[start:pos] - if stashed: - yield stashed - stashed = None - yield (STRING, token, spos, (lnum, pos), line) - else: - strstart = (lnum, start) # multiple lines - contstr = line[start:] - contline = line - break - elif initial in single_quoted or \ - token[:2] in single_quoted or \ - token[:3] in single_quoted: - if token[-1] == '\n': # continued string - strstart = (lnum, start) - endprog = (endprogs[initial] or endprogs[token[1]] or - endprogs[token[2]]) - contstr, needcont = line[start:], 1 - contline = line - break - else: # ordinary string - if stashed: - yield stashed - stashed = None - yield (STRING, token, spos, epos, line) - elif initial.isidentifier(): # ordinary name - if token in ('async', 'await'): - if async_def: - yield (ASYNC if token == 'async' else AWAIT, - token, spos, epos, line) - continue - - tok = (NAME, token, spos, epos, line) - if token == 'async' and not stashed: - stashed = tok - continue - - if token in ('def', 'for'): - if (stashed - and stashed[0] == NAME - and stashed[1] == 'async'): - - if token == 'def': - async_def = True - async_def_indent = indents[-1] - - yield (ASYNC, stashed[1], - stashed[2], stashed[3], - stashed[4]) - stashed = None - - if stashed: - yield stashed - stashed = None - - yield tok - elif initial == '\\': # continued stmt - # This yield is new; needed for better idempotency: - if stashed: - yield stashed - stashed = None - yield (NL, token, spos, (lnum, pos), line) - continued = 1 - else: - if initial in '([{': parenlev = parenlev + 1 - elif initial in ')]}': parenlev = parenlev - 1 - if stashed: - yield stashed - stashed = None - yield (OP, token, spos, epos, line) - else: - yield (ERRORTOKEN, line[pos], - (lnum, pos), (lnum, pos+1), line) - pos = pos + 1 - - if stashed: - yield stashed - stashed = None - - for indent in indents[1:]: # pop remaining indent levels - yield (DEDENT, '', (lnum, 0), (lnum, 0), '') - yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '') - -if __name__ == '__main__': # testing - import sys - if len(sys.argv) > 1: tokenize(open(sys.argv[1]).readline) - else: tokenize(sys.stdin.readline) diff --git a/Lib/lib2to3/pygram.py b/Lib/lib2to3/pygram.py deleted file mode 100644 index 24d9db9217f131..00000000000000 --- a/Lib/lib2to3/pygram.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Export the Python grammar and symbols.""" - -# Python imports -import os - -# Local imports -from .pgen2 import token -from .pgen2 import driver -from . import pytree - -# The grammar file -_GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), "Grammar.txt") -_PATTERN_GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), - "PatternGrammar.txt") - - -class Symbols(object): - - def __init__(self, grammar): - """Initializer. - - Creates an attribute for each grammar symbol (nonterminal), - whose value is the symbol's type (an int >= 256). - """ - for name, symbol in grammar.symbol2number.items(): - setattr(self, name, symbol) - - -python_grammar = driver.load_packaged_grammar("lib2to3", _GRAMMAR_FILE) - -python_symbols = Symbols(python_grammar) - -python_grammar_no_print_statement = python_grammar.copy() -del python_grammar_no_print_statement.keywords["print"] - -python_grammar_no_print_and_exec_statement = python_grammar_no_print_statement.copy() -del python_grammar_no_print_and_exec_statement.keywords["exec"] - -pattern_grammar = driver.load_packaged_grammar("lib2to3", _PATTERN_GRAMMAR_FILE) -pattern_symbols = Symbols(pattern_grammar) diff --git a/Lib/lib2to3/pytree.py b/Lib/lib2to3/pytree.py deleted file mode 100644 index 729023df0284e1..00000000000000 --- a/Lib/lib2to3/pytree.py +++ /dev/null @@ -1,853 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -""" -Python parse tree definitions. - -This is a very concrete parse tree; we need to keep every token and -even the comments and whitespace between tokens. - -There's also a pattern matching implementation here. -""" - -__author__ = "Guido van Rossum " - -import sys -from io import StringIO - -HUGE = 0x7FFFFFFF # maximum repeat count, default max - -_type_reprs = {} -def type_repr(type_num): - global _type_reprs - if not _type_reprs: - from .pygram import python_symbols - # printing tokens is possible but not as useful - # from .pgen2 import token // token.__dict__.items(): - for name, val in python_symbols.__dict__.items(): - if type(val) == int: _type_reprs[val] = name - return _type_reprs.setdefault(type_num, type_num) - -class Base(object): - - """ - Abstract base class for Node and Leaf. - - This provides some default functionality and boilerplate using the - template pattern. - - A node may be a subnode of at most one parent. - """ - - # Default values for instance variables - type = None # int: token number (< 256) or symbol number (>= 256) - parent = None # Parent node pointer, or None - children = () # Tuple of subnodes - was_changed = False - was_checked = False - - def __new__(cls, *args, **kwds): - """Constructor that prevents Base from being instantiated.""" - assert cls is not Base, "Cannot instantiate Base" - return object.__new__(cls) - - def __eq__(self, other): - """ - Compare two nodes for equality. - - This calls the method _eq(). - """ - if self.__class__ is not other.__class__: - return NotImplemented - return self._eq(other) - - __hash__ = None # For Py3 compatibility. - - def _eq(self, other): - """ - Compare two nodes for equality. - - This is called by __eq__ and __ne__. It is only called if the two nodes - have the same type. This must be implemented by the concrete subclass. - Nodes should be considered equal if they have the same structure, - ignoring the prefix string and other context information. - """ - raise NotImplementedError - - def clone(self): - """ - Return a cloned (deep) copy of self. - - This must be implemented by the concrete subclass. - """ - raise NotImplementedError - - def post_order(self): - """ - Return a post-order iterator for the tree. - - This must be implemented by the concrete subclass. - """ - raise NotImplementedError - - def pre_order(self): - """ - Return a pre-order iterator for the tree. - - This must be implemented by the concrete subclass. - """ - raise NotImplementedError - - def replace(self, new): - """Replace this node with a new one in the parent.""" - assert self.parent is not None, str(self) - assert new is not None - if not isinstance(new, list): - new = [new] - l_children = [] - found = False - for ch in self.parent.children: - if ch is self: - assert not found, (self.parent.children, self, new) - if new is not None: - l_children.extend(new) - found = True - else: - l_children.append(ch) - assert found, (self.children, self, new) - self.parent.changed() - self.parent.children = l_children - for x in new: - x.parent = self.parent - self.parent = None - - def get_lineno(self): - """Return the line number which generated the invocant node.""" - node = self - while not isinstance(node, Leaf): - if not node.children: - return - node = node.children[0] - return node.lineno - - def changed(self): - if self.parent: - self.parent.changed() - self.was_changed = True - - def remove(self): - """ - Remove the node from the tree. Returns the position of the node in its - parent's children before it was removed. - """ - if self.parent: - for i, node in enumerate(self.parent.children): - if node is self: - self.parent.changed() - del self.parent.children[i] - self.parent = None - return i - - @property - def next_sibling(self): - """ - The node immediately following the invocant in their parent's children - list. If the invocant does not have a next sibling, it is None - """ - if self.parent is None: - return None - - # Can't use index(); we need to test by identity - for i, child in enumerate(self.parent.children): - if child is self: - try: - return self.parent.children[i+1] - except IndexError: - return None - - @property - def prev_sibling(self): - """ - The node immediately preceding the invocant in their parent's children - list. If the invocant does not have a previous sibling, it is None. - """ - if self.parent is None: - return None - - # Can't use index(); we need to test by identity - for i, child in enumerate(self.parent.children): - if child is self: - if i == 0: - return None - return self.parent.children[i-1] - - def leaves(self): - for child in self.children: - yield from child.leaves() - - def depth(self): - if self.parent is None: - return 0 - return 1 + self.parent.depth() - - def get_suffix(self): - """ - Return the string immediately following the invocant node. This is - effectively equivalent to node.next_sibling.prefix - """ - next_sib = self.next_sibling - if next_sib is None: - return "" - return next_sib.prefix - - if sys.version_info < (3, 0): - def __str__(self): - return str(self).encode("ascii") - -class Node(Base): - - """Concrete implementation for interior nodes.""" - - def __init__(self,type, children, - context=None, - prefix=None, - fixers_applied=None): - """ - Initializer. - - Takes a type constant (a symbol number >= 256), a sequence of - child nodes, and an optional context keyword argument. - - As a side effect, the parent pointers of the children are updated. - """ - assert type >= 256, type - self.type = type - self.children = list(children) - for ch in self.children: - assert ch.parent is None, repr(ch) - ch.parent = self - if prefix is not None: - self.prefix = prefix - if fixers_applied: - self.fixers_applied = fixers_applied[:] - else: - self.fixers_applied = None - - def __repr__(self): - """Return a canonical string representation.""" - return "%s(%s, %r)" % (self.__class__.__name__, - type_repr(self.type), - self.children) - - def __unicode__(self): - """ - Return a pretty string representation. - - This reproduces the input source exactly. - """ - return "".join(map(str, self.children)) - - if sys.version_info > (3, 0): - __str__ = __unicode__ - - def _eq(self, other): - """Compare two nodes for equality.""" - return (self.type, self.children) == (other.type, other.children) - - def clone(self): - """Return a cloned (deep) copy of self.""" - return Node(self.type, [ch.clone() for ch in self.children], - fixers_applied=self.fixers_applied) - - def post_order(self): - """Return a post-order iterator for the tree.""" - for child in self.children: - yield from child.post_order() - yield self - - def pre_order(self): - """Return a pre-order iterator for the tree.""" - yield self - for child in self.children: - yield from child.pre_order() - - @property - def prefix(self): - """ - The whitespace and comments preceding this node in the input. - """ - if not self.children: - return "" - return self.children[0].prefix - - @prefix.setter - def prefix(self, prefix): - if self.children: - self.children[0].prefix = prefix - - def set_child(self, i, child): - """ - Equivalent to 'node.children[i] = child'. This method also sets the - child's parent attribute appropriately. - """ - child.parent = self - self.children[i].parent = None - self.children[i] = child - self.changed() - - def insert_child(self, i, child): - """ - Equivalent to 'node.children.insert(i, child)'. This method also sets - the child's parent attribute appropriately. - """ - child.parent = self - self.children.insert(i, child) - self.changed() - - def append_child(self, child): - """ - Equivalent to 'node.children.append(child)'. This method also sets the - child's parent attribute appropriately. - """ - child.parent = self - self.children.append(child) - self.changed() - - -class Leaf(Base): - - """Concrete implementation for leaf nodes.""" - - # Default values for instance variables - _prefix = "" # Whitespace and comments preceding this token in the input - lineno = 0 # Line where this token starts in the input - column = 0 # Column where this token tarts in the input - - def __init__(self, type, value, - context=None, - prefix=None, - fixers_applied=[]): - """ - Initializer. - - Takes a type constant (a token number < 256), a string value, and an - optional context keyword argument. - """ - assert 0 <= type < 256, type - if context is not None: - self._prefix, (self.lineno, self.column) = context - self.type = type - self.value = value - if prefix is not None: - self._prefix = prefix - self.fixers_applied = fixers_applied[:] - - def __repr__(self): - """Return a canonical string representation.""" - return "%s(%r, %r)" % (self.__class__.__name__, - self.type, - self.value) - - def __unicode__(self): - """ - Return a pretty string representation. - - This reproduces the input source exactly. - """ - return self.prefix + str(self.value) - - if sys.version_info > (3, 0): - __str__ = __unicode__ - - def _eq(self, other): - """Compare two nodes for equality.""" - return (self.type, self.value) == (other.type, other.value) - - def clone(self): - """Return a cloned (deep) copy of self.""" - return Leaf(self.type, self.value, - (self.prefix, (self.lineno, self.column)), - fixers_applied=self.fixers_applied) - - def leaves(self): - yield self - - def post_order(self): - """Return a post-order iterator for the tree.""" - yield self - - def pre_order(self): - """Return a pre-order iterator for the tree.""" - yield self - - @property - def prefix(self): - """ - The whitespace and comments preceding this token in the input. - """ - return self._prefix - - @prefix.setter - def prefix(self, prefix): - self.changed() - self._prefix = prefix - -def convert(gr, raw_node): - """ - Convert raw node information to a Node or Leaf instance. - - This is passed to the parser driver which calls it whenever a reduction of a - grammar rule produces a new complete node, so that the tree is build - strictly bottom-up. - """ - type, value, context, children = raw_node - if children or type in gr.number2symbol: - # If there's exactly one child, return that child instead of - # creating a new node. - if len(children) == 1: - return children[0] - return Node(type, children, context=context) - else: - return Leaf(type, value, context=context) - - -class BasePattern(object): - - """ - A pattern is a tree matching pattern. - - It looks for a specific node type (token or symbol), and - optionally for a specific content. - - This is an abstract base class. There are three concrete - subclasses: - - - LeafPattern matches a single leaf node; - - NodePattern matches a single node (usually non-leaf); - - WildcardPattern matches a sequence of nodes of variable length. - """ - - # Defaults for instance variables - type = None # Node type (token if < 256, symbol if >= 256) - content = None # Optional content matching pattern - name = None # Optional name used to store match in results dict - - def __new__(cls, *args, **kwds): - """Constructor that prevents BasePattern from being instantiated.""" - assert cls is not BasePattern, "Cannot instantiate BasePattern" - return object.__new__(cls) - - def __repr__(self): - args = [type_repr(self.type), self.content, self.name] - while args and args[-1] is None: - del args[-1] - return "%s(%s)" % (self.__class__.__name__, ", ".join(map(repr, args))) - - def optimize(self): - """ - A subclass can define this as a hook for optimizations. - - Returns either self or another node with the same effect. - """ - return self - - def match(self, node, results=None): - """ - Does this pattern exactly match a node? - - Returns True if it matches, False if not. - - If results is not None, it must be a dict which will be - updated with the nodes matching named subpatterns. - - Default implementation for non-wildcard patterns. - """ - if self.type is not None and node.type != self.type: - return False - if self.content is not None: - r = None - if results is not None: - r = {} - if not self._submatch(node, r): - return False - if r: - results.update(r) - if results is not None and self.name: - results[self.name] = node - return True - - def match_seq(self, nodes, results=None): - """ - Does this pattern exactly match a sequence of nodes? - - Default implementation for non-wildcard patterns. - """ - if len(nodes) != 1: - return False - return self.match(nodes[0], results) - - def generate_matches(self, nodes): - """ - Generator yielding all matches for this pattern. - - Default implementation for non-wildcard patterns. - """ - r = {} - if nodes and self.match(nodes[0], r): - yield 1, r - - -class LeafPattern(BasePattern): - - def __init__(self, type=None, content=None, name=None): - """ - Initializer. Takes optional type, content, and name. - - The type, if given must be a token type (< 256). If not given, - this matches any *leaf* node; the content may still be required. - - The content, if given, must be a string. - - If a name is given, the matching node is stored in the results - dict under that key. - """ - if type is not None: - assert 0 <= type < 256, type - if content is not None: - assert isinstance(content, str), repr(content) - self.type = type - self.content = content - self.name = name - - def match(self, node, results=None): - """Override match() to insist on a leaf node.""" - if not isinstance(node, Leaf): - return False - return BasePattern.match(self, node, results) - - def _submatch(self, node, results=None): - """ - Match the pattern's content to the node's children. - - This assumes the node type matches and self.content is not None. - - Returns True if it matches, False if not. - - If results is not None, it must be a dict which will be - updated with the nodes matching named subpatterns. - - When returning False, the results dict may still be updated. - """ - return self.content == node.value - - -class NodePattern(BasePattern): - - wildcards = False - - def __init__(self, type=None, content=None, name=None): - """ - Initializer. Takes optional type, content, and name. - - The type, if given, must be a symbol type (>= 256). If the - type is None this matches *any* single node (leaf or not), - except if content is not None, in which it only matches - non-leaf nodes that also match the content pattern. - - The content, if not None, must be a sequence of Patterns that - must match the node's children exactly. If the content is - given, the type must not be None. - - If a name is given, the matching node is stored in the results - dict under that key. - """ - if type is not None: - assert type >= 256, type - if content is not None: - assert not isinstance(content, str), repr(content) - content = list(content) - for i, item in enumerate(content): - assert isinstance(item, BasePattern), (i, item) - if isinstance(item, WildcardPattern): - self.wildcards = True - self.type = type - self.content = content - self.name = name - - def _submatch(self, node, results=None): - """ - Match the pattern's content to the node's children. - - This assumes the node type matches and self.content is not None. - - Returns True if it matches, False if not. - - If results is not None, it must be a dict which will be - updated with the nodes matching named subpatterns. - - When returning False, the results dict may still be updated. - """ - if self.wildcards: - for c, r in generate_matches(self.content, node.children): - if c == len(node.children): - if results is not None: - results.update(r) - return True - return False - if len(self.content) != len(node.children): - return False - for subpattern, child in zip(self.content, node.children): - if not subpattern.match(child, results): - return False - return True - - -class WildcardPattern(BasePattern): - - """ - A wildcard pattern can match zero or more nodes. - - This has all the flexibility needed to implement patterns like: - - .* .+ .? .{m,n} - (a b c | d e | f) - (...)* (...)+ (...)? (...){m,n} - - except it always uses non-greedy matching. - """ - - def __init__(self, content=None, min=0, max=HUGE, name=None): - """ - Initializer. - - Args: - content: optional sequence of subsequences of patterns; - if absent, matches one node; - if present, each subsequence is an alternative [*] - min: optional minimum number of times to match, default 0 - max: optional maximum number of times to match, default HUGE - name: optional name assigned to this match - - [*] Thus, if content is [[a, b, c], [d, e], [f, g, h]] this is - equivalent to (a b c | d e | f g h); if content is None, - this is equivalent to '.' in regular expression terms. - The min and max parameters work as follows: - min=0, max=maxint: .* - min=1, max=maxint: .+ - min=0, max=1: .? - min=1, max=1: . - If content is not None, replace the dot with the parenthesized - list of alternatives, e.g. (a b c | d e | f g h)* - """ - assert 0 <= min <= max <= HUGE, (min, max) - if content is not None: - content = tuple(map(tuple, content)) # Protect against alterations - # Check sanity of alternatives - assert len(content), repr(content) # Can't have zero alternatives - for alt in content: - assert len(alt), repr(alt) # Can have empty alternatives - self.content = content - self.min = min - self.max = max - self.name = name - - def optimize(self): - """Optimize certain stacked wildcard patterns.""" - subpattern = None - if (self.content is not None and - len(self.content) == 1 and len(self.content[0]) == 1): - subpattern = self.content[0][0] - if self.min == 1 and self.max == 1: - if self.content is None: - return NodePattern(name=self.name) - if subpattern is not None and self.name == subpattern.name: - return subpattern.optimize() - if (self.min <= 1 and isinstance(subpattern, WildcardPattern) and - subpattern.min <= 1 and self.name == subpattern.name): - return WildcardPattern(subpattern.content, - self.min*subpattern.min, - self.max*subpattern.max, - subpattern.name) - return self - - def match(self, node, results=None): - """Does this pattern exactly match a node?""" - return self.match_seq([node], results) - - def match_seq(self, nodes, results=None): - """Does this pattern exactly match a sequence of nodes?""" - for c, r in self.generate_matches(nodes): - if c == len(nodes): - if results is not None: - results.update(r) - if self.name: - results[self.name] = list(nodes) - return True - return False - - def generate_matches(self, nodes): - """ - Generator yielding matches for a sequence of nodes. - - Args: - nodes: sequence of nodes - - Yields: - (count, results) tuples where: - count: the match comprises nodes[:count]; - results: dict containing named submatches. - """ - if self.content is None: - # Shortcut for special case (see __init__.__doc__) - for count in range(self.min, 1 + min(len(nodes), self.max)): - r = {} - if self.name: - r[self.name] = nodes[:count] - yield count, r - elif self.name == "bare_name": - yield self._bare_name_matches(nodes) - else: - # The reason for this is that hitting the recursion limit usually - # results in some ugly messages about how RuntimeErrors are being - # ignored. We only have to do this on CPython, though, because other - # implementations don't have this nasty bug in the first place. - if hasattr(sys, "getrefcount"): - save_stderr = sys.stderr - sys.stderr = StringIO() - try: - for count, r in self._recursive_matches(nodes, 0): - if self.name: - r[self.name] = nodes[:count] - yield count, r - except RuntimeError: - # Fall back to the iterative pattern matching scheme if the - # recursive scheme hits the recursion limit (RecursionError). - for count, r in self._iterative_matches(nodes): - if self.name: - r[self.name] = nodes[:count] - yield count, r - finally: - if hasattr(sys, "getrefcount"): - sys.stderr = save_stderr - - def _iterative_matches(self, nodes): - """Helper to iteratively yield the matches.""" - nodelen = len(nodes) - if 0 >= self.min: - yield 0, {} - - results = [] - # generate matches that use just one alt from self.content - for alt in self.content: - for c, r in generate_matches(alt, nodes): - yield c, r - results.append((c, r)) - - # for each match, iterate down the nodes - while results: - new_results = [] - for c0, r0 in results: - # stop if the entire set of nodes has been matched - if c0 < nodelen and c0 <= self.max: - for alt in self.content: - for c1, r1 in generate_matches(alt, nodes[c0:]): - if c1 > 0: - r = {} - r.update(r0) - r.update(r1) - yield c0 + c1, r - new_results.append((c0 + c1, r)) - results = new_results - - def _bare_name_matches(self, nodes): - """Special optimized matcher for bare_name.""" - count = 0 - r = {} - done = False - max = len(nodes) - while not done and count < max: - done = True - for leaf in self.content: - if leaf[0].match(nodes[count], r): - count += 1 - done = False - break - r[self.name] = nodes[:count] - return count, r - - def _recursive_matches(self, nodes, count): - """Helper to recursively yield the matches.""" - assert self.content is not None - if count >= self.min: - yield 0, {} - if count < self.max: - for alt in self.content: - for c0, r0 in generate_matches(alt, nodes): - for c1, r1 in self._recursive_matches(nodes[c0:], count+1): - r = {} - r.update(r0) - r.update(r1) - yield c0 + c1, r - - -class NegatedPattern(BasePattern): - - def __init__(self, content=None): - """ - Initializer. - - The argument is either a pattern or None. If it is None, this - only matches an empty sequence (effectively '$' in regex - lingo). If it is not None, this matches whenever the argument - pattern doesn't have any matches. - """ - if content is not None: - assert isinstance(content, BasePattern), repr(content) - self.content = content - - def match(self, node): - # We never match a node in its entirety - return False - - def match_seq(self, nodes): - # We only match an empty sequence of nodes in its entirety - return len(nodes) == 0 - - def generate_matches(self, nodes): - if self.content is None: - # Return a match if there is an empty sequence - if len(nodes) == 0: - yield 0, {} - else: - # Return a match if the argument pattern has no matches - for c, r in self.content.generate_matches(nodes): - return - yield 0, {} - - -def generate_matches(patterns, nodes): - """ - Generator yielding matches for a sequence of patterns and nodes. - - Args: - patterns: a sequence of patterns - nodes: a sequence of nodes - - Yields: - (count, results) tuples where: - count: the entire sequence of patterns matches nodes[:count]; - results: dict containing named submatches. - """ - if not patterns: - yield 0, {} - else: - p, rest = patterns[0], patterns[1:] - for c0, r0 in p.generate_matches(nodes): - if not rest: - yield c0, r0 - else: - for c1, r1 in generate_matches(rest, nodes[c0:]): - r = {} - r.update(r0) - r.update(r1) - yield c0 + c1, r diff --git a/Lib/lib2to3/refactor.py b/Lib/lib2to3/refactor.py deleted file mode 100644 index 3a5aafffc6df07..00000000000000 --- a/Lib/lib2to3/refactor.py +++ /dev/null @@ -1,732 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Refactoring framework. - -Used as a main program, this can refactor any number of files and/or -recursively descend down directories. Imported as a module, this -provides infrastructure to write your own refactoring tool. -""" - -__author__ = "Guido van Rossum " - - -# Python imports -import io -import os -import pkgutil -import sys -import logging -import operator -import collections -from itertools import chain - -# Local imports -from .pgen2 import driver, tokenize, token -from .fixer_util import find_root -from . import pytree, pygram -from . import btm_matcher as bm - - -def get_all_fix_names(fixer_pkg, remove_prefix=True): - """Return a sorted list of all available fix names in the given package.""" - pkg = __import__(fixer_pkg, [], [], ["*"]) - fix_names = [] - for finder, name, ispkg in pkgutil.iter_modules(pkg.__path__): - if name.startswith("fix_"): - if remove_prefix: - name = name[4:] - fix_names.append(name) - return fix_names - - -class _EveryNode(Exception): - pass - - -def _get_head_types(pat): - """ Accepts a pytree Pattern Node and returns a set - of the pattern types which will match first. """ - - if isinstance(pat, (pytree.NodePattern, pytree.LeafPattern)): - # NodePatters must either have no type and no content - # or a type and content -- so they don't get any farther - # Always return leafs - if pat.type is None: - raise _EveryNode - return {pat.type} - - if isinstance(pat, pytree.NegatedPattern): - if pat.content: - return _get_head_types(pat.content) - raise _EveryNode # Negated Patterns don't have a type - - if isinstance(pat, pytree.WildcardPattern): - # Recurse on each node in content - r = set() - for p in pat.content: - for x in p: - r.update(_get_head_types(x)) - return r - - raise Exception("Oh no! I don't understand pattern %s" %(pat)) - - -def _get_headnode_dict(fixer_list): - """ Accepts a list of fixers and returns a dictionary - of head node type --> fixer list. """ - head_nodes = collections.defaultdict(list) - every = [] - for fixer in fixer_list: - if fixer.pattern: - try: - heads = _get_head_types(fixer.pattern) - except _EveryNode: - every.append(fixer) - else: - for node_type in heads: - head_nodes[node_type].append(fixer) - else: - if fixer._accept_type is not None: - head_nodes[fixer._accept_type].append(fixer) - else: - every.append(fixer) - for node_type in chain(pygram.python_grammar.symbol2number.values(), - pygram.python_grammar.tokens): - head_nodes[node_type].extend(every) - return dict(head_nodes) - - -def get_fixers_from_package(pkg_name): - """ - Return the fully qualified names for fixers in the package pkg_name. - """ - return [pkg_name + "." + fix_name - for fix_name in get_all_fix_names(pkg_name, False)] - -def _identity(obj): - return obj - - -def _detect_future_features(source): - have_docstring = False - gen = tokenize.generate_tokens(io.StringIO(source).readline) - def advance(): - tok = next(gen) - return tok[0], tok[1] - ignore = frozenset({token.NEWLINE, tokenize.NL, token.COMMENT}) - features = set() - try: - while True: - tp, value = advance() - if tp in ignore: - continue - elif tp == token.STRING: - if have_docstring: - break - have_docstring = True - elif tp == token.NAME and value == "from": - tp, value = advance() - if tp != token.NAME or value != "__future__": - break - tp, value = advance() - if tp != token.NAME or value != "import": - break - tp, value = advance() - if tp == token.OP and value == "(": - tp, value = advance() - while tp == token.NAME: - features.add(value) - tp, value = advance() - if tp != token.OP or value != ",": - break - tp, value = advance() - else: - break - except StopIteration: - pass - return frozenset(features) - - -class FixerError(Exception): - """A fixer could not be loaded.""" - - -class RefactoringTool(object): - - _default_options = {"print_function" : False, - "exec_function": False, - "write_unchanged_files" : False} - - CLASS_PREFIX = "Fix" # The prefix for fixer classes - FILE_PREFIX = "fix_" # The prefix for modules with a fixer within - - def __init__(self, fixer_names, options=None, explicit=None): - """Initializer. - - Args: - fixer_names: a list of fixers to import - options: a dict with configuration. - explicit: a list of fixers to run even if they are explicit. - """ - self.fixers = fixer_names - self.explicit = explicit or [] - self.options = self._default_options.copy() - if options is not None: - self.options.update(options) - self.grammar = pygram.python_grammar.copy() - - if self.options['print_function']: - del self.grammar.keywords["print"] - elif self.options['exec_function']: - del self.grammar.keywords["exec"] - - # When this is True, the refactor*() methods will call write_file() for - # files processed even if they were not changed during refactoring. If - # and only if the refactor method's write parameter was True. - self.write_unchanged_files = self.options.get("write_unchanged_files") - self.errors = [] - self.logger = logging.getLogger("RefactoringTool") - self.fixer_log = [] - self.wrote = False - self.driver = driver.Driver(self.grammar, - convert=pytree.convert, - logger=self.logger) - self.pre_order, self.post_order = self.get_fixers() - - - self.files = [] # List of files that were or should be modified - - self.BM = bm.BottomMatcher() - self.bmi_pre_order = [] # Bottom Matcher incompatible fixers - self.bmi_post_order = [] - - for fixer in chain(self.post_order, self.pre_order): - if fixer.BM_compatible: - self.BM.add_fixer(fixer) - # remove fixers that will be handled by the bottom-up - # matcher - elif fixer in self.pre_order: - self.bmi_pre_order.append(fixer) - elif fixer in self.post_order: - self.bmi_post_order.append(fixer) - - self.bmi_pre_order_heads = _get_headnode_dict(self.bmi_pre_order) - self.bmi_post_order_heads = _get_headnode_dict(self.bmi_post_order) - - - - def get_fixers(self): - """Inspects the options to load the requested patterns and handlers. - - Returns: - (pre_order, post_order), where pre_order is the list of fixers that - want a pre-order AST traversal, and post_order is the list that want - post-order traversal. - """ - pre_order_fixers = [] - post_order_fixers = [] - for fix_mod_path in self.fixers: - mod = __import__(fix_mod_path, {}, {}, ["*"]) - fix_name = fix_mod_path.rsplit(".", 1)[-1] - if fix_name.startswith(self.FILE_PREFIX): - fix_name = fix_name[len(self.FILE_PREFIX):] - parts = fix_name.split("_") - class_name = self.CLASS_PREFIX + "".join([p.title() for p in parts]) - try: - fix_class = getattr(mod, class_name) - except AttributeError: - raise FixerError("Can't find %s.%s" % (fix_name, class_name)) from None - fixer = fix_class(self.options, self.fixer_log) - if fixer.explicit and self.explicit is not True and \ - fix_mod_path not in self.explicit: - self.log_message("Skipping optional fixer: %s", fix_name) - continue - - self.log_debug("Adding transformation: %s", fix_name) - if fixer.order == "pre": - pre_order_fixers.append(fixer) - elif fixer.order == "post": - post_order_fixers.append(fixer) - else: - raise FixerError("Illegal fixer order: %r" % fixer.order) - - key_func = operator.attrgetter("run_order") - pre_order_fixers.sort(key=key_func) - post_order_fixers.sort(key=key_func) - return (pre_order_fixers, post_order_fixers) - - def log_error(self, msg, *args, **kwds): - """Called when an error occurs.""" - raise - - def log_message(self, msg, *args): - """Hook to log a message.""" - if args: - msg = msg % args - self.logger.info(msg) - - def log_debug(self, msg, *args): - if args: - msg = msg % args - self.logger.debug(msg) - - def print_output(self, old_text, new_text, filename, equal): - """Called with the old version, new version, and filename of a - refactored file.""" - pass - - def refactor(self, items, write=False, doctests_only=False): - """Refactor a list of files and directories.""" - - for dir_or_file in items: - if os.path.isdir(dir_or_file): - self.refactor_dir(dir_or_file, write, doctests_only) - else: - self.refactor_file(dir_or_file, write, doctests_only) - - def refactor_dir(self, dir_name, write=False, doctests_only=False): - """Descends down a directory and refactor every Python file found. - - Python files are assumed to have a .py extension. - - Files and subdirectories starting with '.' are skipped. - """ - py_ext = os.extsep + "py" - for dirpath, dirnames, filenames in os.walk(dir_name): - self.log_debug("Descending into %s", dirpath) - dirnames.sort() - filenames.sort() - for name in filenames: - if (not name.startswith(".") and - os.path.splitext(name)[1] == py_ext): - fullname = os.path.join(dirpath, name) - self.refactor_file(fullname, write, doctests_only) - # Modify dirnames in-place to remove subdirs with leading dots - dirnames[:] = [dn for dn in dirnames if not dn.startswith(".")] - - def _read_python_source(self, filename): - """ - Do our best to decode a Python source file correctly. - """ - try: - f = open(filename, "rb") - except OSError as err: - self.log_error("Can't open %s: %s", filename, err) - return None, None - try: - encoding = tokenize.detect_encoding(f.readline)[0] - finally: - f.close() - with io.open(filename, "r", encoding=encoding, newline='') as f: - return f.read(), encoding - - def refactor_file(self, filename, write=False, doctests_only=False): - """Refactors a file.""" - input, encoding = self._read_python_source(filename) - if input is None: - # Reading the file failed. - return - input += "\n" # Silence certain parse errors - if doctests_only: - self.log_debug("Refactoring doctests in %s", filename) - output = self.refactor_docstring(input, filename) - if self.write_unchanged_files or output != input: - self.processed_file(output, filename, input, write, encoding) - else: - self.log_debug("No doctest changes in %s", filename) - else: - tree = self.refactor_string(input, filename) - if self.write_unchanged_files or (tree and tree.was_changed): - # The [:-1] is to take off the \n we added earlier - self.processed_file(str(tree)[:-1], filename, - write=write, encoding=encoding) - else: - self.log_debug("No changes in %s", filename) - - def refactor_string(self, data, name): - """Refactor a given input string. - - Args: - data: a string holding the code to be refactored. - name: a human-readable name for use in error/log messages. - - Returns: - An AST corresponding to the refactored input stream; None if - there were errors during the parse. - """ - features = _detect_future_features(data) - if "print_function" in features: - self.driver.grammar = pygram.python_grammar_no_print_statement - try: - tree = self.driver.parse_string(data) - except Exception as err: - self.log_error("Can't parse %s: %s: %s", - name, err.__class__.__name__, err) - return - finally: - self.driver.grammar = self.grammar - tree.future_features = features - self.log_debug("Refactoring %s", name) - self.refactor_tree(tree, name) - return tree - - def refactor_stdin(self, doctests_only=False): - input = sys.stdin.read() - if doctests_only: - self.log_debug("Refactoring doctests in stdin") - output = self.refactor_docstring(input, "") - if self.write_unchanged_files or output != input: - self.processed_file(output, "", input) - else: - self.log_debug("No doctest changes in stdin") - else: - tree = self.refactor_string(input, "") - if self.write_unchanged_files or (tree and tree.was_changed): - self.processed_file(str(tree), "", input) - else: - self.log_debug("No changes in stdin") - - def refactor_tree(self, tree, name): - """Refactors a parse tree (modifying the tree in place). - - For compatible patterns the bottom matcher module is - used. Otherwise the tree is traversed node-to-node for - matches. - - Args: - tree: a pytree.Node instance representing the root of the tree - to be refactored. - name: a human-readable name for this tree. - - Returns: - True if the tree was modified, False otherwise. - """ - - for fixer in chain(self.pre_order, self.post_order): - fixer.start_tree(tree, name) - - #use traditional matching for the incompatible fixers - self.traverse_by(self.bmi_pre_order_heads, tree.pre_order()) - self.traverse_by(self.bmi_post_order_heads, tree.post_order()) - - # obtain a set of candidate nodes - match_set = self.BM.run(tree.leaves()) - - while any(match_set.values()): - for fixer in self.BM.fixers: - if fixer in match_set and match_set[fixer]: - #sort by depth; apply fixers from bottom(of the AST) to top - match_set[fixer].sort(key=pytree.Base.depth, reverse=True) - - if fixer.keep_line_order: - #some fixers(eg fix_imports) must be applied - #with the original file's line order - match_set[fixer].sort(key=pytree.Base.get_lineno) - - for node in list(match_set[fixer]): - if node in match_set[fixer]: - match_set[fixer].remove(node) - - try: - find_root(node) - except ValueError: - # this node has been cut off from a - # previous transformation ; skip - continue - - if node.fixers_applied and fixer in node.fixers_applied: - # do not apply the same fixer again - continue - - results = fixer.match(node) - - if results: - new = fixer.transform(node, results) - if new is not None: - node.replace(new) - #new.fixers_applied.append(fixer) - for node in new.post_order(): - # do not apply the fixer again to - # this or any subnode - if not node.fixers_applied: - node.fixers_applied = [] - node.fixers_applied.append(fixer) - - # update the original match set for - # the added code - new_matches = self.BM.run(new.leaves()) - for fxr in new_matches: - if not fxr in match_set: - match_set[fxr]=[] - - match_set[fxr].extend(new_matches[fxr]) - - for fixer in chain(self.pre_order, self.post_order): - fixer.finish_tree(tree, name) - return tree.was_changed - - def traverse_by(self, fixers, traversal): - """Traverse an AST, applying a set of fixers to each node. - - This is a helper method for refactor_tree(). - - Args: - fixers: a list of fixer instances. - traversal: a generator that yields AST nodes. - - Returns: - None - """ - if not fixers: - return - for node in traversal: - for fixer in fixers[node.type]: - results = fixer.match(node) - if results: - new = fixer.transform(node, results) - if new is not None: - node.replace(new) - node = new - - def processed_file(self, new_text, filename, old_text=None, write=False, - encoding=None): - """ - Called when a file has been refactored and there may be changes. - """ - self.files.append(filename) - if old_text is None: - old_text = self._read_python_source(filename)[0] - if old_text is None: - return - equal = old_text == new_text - self.print_output(old_text, new_text, filename, equal) - if equal: - self.log_debug("No changes to %s", filename) - if not self.write_unchanged_files: - return - if write: - self.write_file(new_text, filename, old_text, encoding) - else: - self.log_debug("Not writing changes to %s", filename) - - def write_file(self, new_text, filename, old_text, encoding=None): - """Writes a string to a file. - - It first shows a unified diff between the old text and the new text, and - then rewrites the file; the latter is only done if the write option is - set. - """ - try: - fp = io.open(filename, "w", encoding=encoding, newline='') - except OSError as err: - self.log_error("Can't create %s: %s", filename, err) - return - - with fp: - try: - fp.write(new_text) - except OSError as err: - self.log_error("Can't write %s: %s", filename, err) - self.log_debug("Wrote changes to %s", filename) - self.wrote = True - - PS1 = ">>> " - PS2 = "... " - - def refactor_docstring(self, input, filename): - """Refactors a docstring, looking for doctests. - - This returns a modified version of the input string. It looks - for doctests, which start with a ">>>" prompt, and may be - continued with "..." prompts, as long as the "..." is indented - the same as the ">>>". - - (Unfortunately we can't use the doctest module's parser, - since, like most parsers, it is not geared towards preserving - the original source.) - """ - result = [] - block = None - block_lineno = None - indent = None - lineno = 0 - for line in input.splitlines(keepends=True): - lineno += 1 - if line.lstrip().startswith(self.PS1): - if block is not None: - result.extend(self.refactor_doctest(block, block_lineno, - indent, filename)) - block_lineno = lineno - block = [line] - i = line.find(self.PS1) - indent = line[:i] - elif (indent is not None and - (line.startswith(indent + self.PS2) or - line == indent + self.PS2.rstrip() + "\n")): - block.append(line) - else: - if block is not None: - result.extend(self.refactor_doctest(block, block_lineno, - indent, filename)) - block = None - indent = None - result.append(line) - if block is not None: - result.extend(self.refactor_doctest(block, block_lineno, - indent, filename)) - return "".join(result) - - def refactor_doctest(self, block, lineno, indent, filename): - """Refactors one doctest. - - A doctest is given as a block of lines, the first of which starts - with ">>>" (possibly indented), while the remaining lines start - with "..." (identically indented). - - """ - try: - tree = self.parse_block(block, lineno, indent) - except Exception as err: - if self.logger.isEnabledFor(logging.DEBUG): - for line in block: - self.log_debug("Source: %s", line.rstrip("\n")) - self.log_error("Can't parse docstring in %s line %s: %s: %s", - filename, lineno, err.__class__.__name__, err) - return block - if self.refactor_tree(tree, filename): - new = str(tree).splitlines(keepends=True) - # Undo the adjustment of the line numbers in wrap_toks() below. - clipped, new = new[:lineno-1], new[lineno-1:] - assert clipped == ["\n"] * (lineno-1), clipped - if not new[-1].endswith("\n"): - new[-1] += "\n" - block = [indent + self.PS1 + new.pop(0)] - if new: - block += [indent + self.PS2 + line for line in new] - return block - - def summarize(self): - if self.wrote: - were = "were" - else: - were = "need to be" - if not self.files: - self.log_message("No files %s modified.", were) - else: - self.log_message("Files that %s modified:", were) - for file in self.files: - self.log_message(file) - if self.fixer_log: - self.log_message("Warnings/messages while refactoring:") - for message in self.fixer_log: - self.log_message(message) - if self.errors: - if len(self.errors) == 1: - self.log_message("There was 1 error:") - else: - self.log_message("There were %d errors:", len(self.errors)) - for msg, args, kwds in self.errors: - self.log_message(msg, *args, **kwds) - - def parse_block(self, block, lineno, indent): - """Parses a block into a tree. - - This is necessary to get correct line number / offset information - in the parser diagnostics and embedded into the parse tree. - """ - tree = self.driver.parse_tokens(self.wrap_toks(block, lineno, indent)) - tree.future_features = frozenset() - return tree - - def wrap_toks(self, block, lineno, indent): - """Wraps a tokenize stream to systematically modify start/end.""" - tokens = tokenize.generate_tokens(self.gen_lines(block, indent).__next__) - for type, value, (line0, col0), (line1, col1), line_text in tokens: - line0 += lineno - 1 - line1 += lineno - 1 - # Don't bother updating the columns; this is too complicated - # since line_text would also have to be updated and it would - # still break for tokens spanning lines. Let the user guess - # that the column numbers for doctests are relative to the - # end of the prompt string (PS1 or PS2). - yield type, value, (line0, col0), (line1, col1), line_text - - - def gen_lines(self, block, indent): - """Generates lines as expected by tokenize from a list of lines. - - This strips the first len(indent + self.PS1) characters off each line. - """ - prefix1 = indent + self.PS1 - prefix2 = indent + self.PS2 - prefix = prefix1 - for line in block: - if line.startswith(prefix): - yield line[len(prefix):] - elif line == prefix.rstrip() + "\n": - yield "\n" - else: - raise AssertionError("line=%r, prefix=%r" % (line, prefix)) - prefix = prefix2 - while True: - yield "" - - -class MultiprocessingUnsupported(Exception): - pass - - -class MultiprocessRefactoringTool(RefactoringTool): - - def __init__(self, *args, **kwargs): - super(MultiprocessRefactoringTool, self).__init__(*args, **kwargs) - self.queue = None - self.output_lock = None - - def refactor(self, items, write=False, doctests_only=False, - num_processes=1): - if num_processes == 1: - return super(MultiprocessRefactoringTool, self).refactor( - items, write, doctests_only) - try: - import multiprocessing - except ImportError: - raise MultiprocessingUnsupported - if self.queue is not None: - raise RuntimeError("already doing multiple processes") - self.queue = multiprocessing.JoinableQueue() - self.output_lock = multiprocessing.Lock() - processes = [multiprocessing.Process(target=self._child) - for i in range(num_processes)] - try: - for p in processes: - p.start() - super(MultiprocessRefactoringTool, self).refactor(items, write, - doctests_only) - finally: - self.queue.join() - for i in range(num_processes): - self.queue.put(None) - for p in processes: - if p.is_alive(): - p.join() - self.queue = None - - def _child(self): - task = self.queue.get() - while task is not None: - args, kwargs = task - try: - super(MultiprocessRefactoringTool, self).refactor_file( - *args, **kwargs) - finally: - self.queue.task_done() - task = self.queue.get() - - def refactor_file(self, *args, **kwargs): - if self.queue is not None: - self.queue.put((args, kwargs)) - else: - return super(MultiprocessRefactoringTool, self).refactor_file( - *args, **kwargs) diff --git a/Lib/test/test_lib2to3/__init__.py b/Lib/test/test_lib2to3/__init__.py deleted file mode 100644 index f323c2355b2287..00000000000000 --- a/Lib/test/test_lib2to3/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -# Author: Collin Winter - -import os -import warnings - -from test.support import load_package_tests - -def load_tests(*args): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', category=DeprecationWarning, message='lib2to3') - return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_lib2to3/__main__.py b/Lib/test/test_lib2to3/__main__.py deleted file mode 100644 index 40a23a297ec2b4..00000000000000 --- a/Lib/test/test_lib2to3/__main__.py +++ /dev/null @@ -1,4 +0,0 @@ -from . import load_tests -import unittest - -unittest.main() diff --git a/Lib/test/test_lib2to3/data/README b/Lib/test/test_lib2to3/data/README deleted file mode 100644 index 7aa47e40aa97f5..00000000000000 --- a/Lib/test/test_lib2to3/data/README +++ /dev/null @@ -1,6 +0,0 @@ -In this directory: -- py2_test_grammar.py -- test file that exercises most/all of Python 2.x's grammar. -- py3_test_grammar.py -- test file that exercises most/all of Python 3.x's grammar. -- infinite_recursion.py -- test file that causes lib2to3's faster recursive pattern matching - scheme to fail, but passes when lib2to3 falls back to iterative pattern matching. -- fixes/ -- for use by test_refactor.py diff --git a/Lib/test/test_lib2to3/data/bom.py b/Lib/test/test_lib2to3/data/bom.py deleted file mode 100644 index 9bc3975a42f601..00000000000000 --- a/Lib/test/test_lib2to3/data/bom.py +++ /dev/null @@ -1,2 +0,0 @@ -# coding: utf-8 -print "BOM BOOM!" diff --git a/Lib/test/test_lib2to3/data/crlf.py b/Lib/test/test_lib2to3/data/crlf.py deleted file mode 100644 index a83ca8f0a2068f..00000000000000 --- a/Lib/test/test_lib2to3/data/crlf.py +++ /dev/null @@ -1,3 +0,0 @@ -print "hi" - -print "Like bad Windows newlines?" diff --git a/Lib/test/test_lib2to3/data/different_encoding.py b/Lib/test/test_lib2to3/data/different_encoding.py deleted file mode 100755 index 9f32bd04dc28db..00000000000000 --- a/Lib/test/test_lib2to3/data/different_encoding.py +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -print u'ßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ' - -def f(x): - print '%s\t-> α(%2i):%s β(%s)' diff --git a/Lib/test/test_lib2to3/data/false_encoding.py b/Lib/test/test_lib2to3/data/false_encoding.py deleted file mode 100755 index f4e59e787da1e5..00000000000000 --- a/Lib/test/test_lib2to3/data/false_encoding.py +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env python -print '#coding=0' diff --git a/Lib/test/test_lib2to3/data/fixers/bad_order.py b/Lib/test/test_lib2to3/data/fixers/bad_order.py deleted file mode 100644 index 061bbf209a0dee..00000000000000 --- a/Lib/test/test_lib2to3/data/fixers/bad_order.py +++ /dev/null @@ -1,5 +0,0 @@ -from lib2to3.fixer_base import BaseFix - -class FixBadOrder(BaseFix): - - order = "crazy" diff --git a/Lib/test/test_lib2to3/data/fixers/myfixes/__init__.py b/Lib/test/test_lib2to3/data/fixers/myfixes/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_lib2to3/data/fixers/myfixes/fix_explicit.py b/Lib/test/test_lib2to3/data/fixers/myfixes/fix_explicit.py deleted file mode 100644 index cbe16f6f1f0dfc..00000000000000 --- a/Lib/test/test_lib2to3/data/fixers/myfixes/fix_explicit.py +++ /dev/null @@ -1,6 +0,0 @@ -from lib2to3.fixer_base import BaseFix - -class FixExplicit(BaseFix): - explicit = True - - def match(self): return False diff --git a/Lib/test/test_lib2to3/data/fixers/myfixes/fix_first.py b/Lib/test/test_lib2to3/data/fixers/myfixes/fix_first.py deleted file mode 100644 index a88821f06b2d14..00000000000000 --- a/Lib/test/test_lib2to3/data/fixers/myfixes/fix_first.py +++ /dev/null @@ -1,6 +0,0 @@ -from lib2to3.fixer_base import BaseFix - -class FixFirst(BaseFix): - run_order = 1 - - def match(self, node): return False diff --git a/Lib/test/test_lib2to3/data/fixers/myfixes/fix_last.py b/Lib/test/test_lib2to3/data/fixers/myfixes/fix_last.py deleted file mode 100644 index 9a077d41594da6..00000000000000 --- a/Lib/test/test_lib2to3/data/fixers/myfixes/fix_last.py +++ /dev/null @@ -1,7 +0,0 @@ -from lib2to3.fixer_base import BaseFix - -class FixLast(BaseFix): - - run_order = 10 - - def match(self, node): return False diff --git a/Lib/test/test_lib2to3/data/fixers/myfixes/fix_parrot.py b/Lib/test/test_lib2to3/data/fixers/myfixes/fix_parrot.py deleted file mode 100644 index 6db79adf8d80c1..00000000000000 --- a/Lib/test/test_lib2to3/data/fixers/myfixes/fix_parrot.py +++ /dev/null @@ -1,13 +0,0 @@ -from lib2to3.fixer_base import BaseFix -from lib2to3.fixer_util import Name - -class FixParrot(BaseFix): - """ - Change functions named 'parrot' to 'cheese'. - """ - - PATTERN = """funcdef < 'def' name='parrot' any* >""" - - def transform(self, node, results): - name = results["name"] - name.replace(Name("cheese", name.prefix)) diff --git a/Lib/test/test_lib2to3/data/fixers/myfixes/fix_preorder.py b/Lib/test/test_lib2to3/data/fixers/myfixes/fix_preorder.py deleted file mode 100644 index b9bfbba9d5e900..00000000000000 --- a/Lib/test/test_lib2to3/data/fixers/myfixes/fix_preorder.py +++ /dev/null @@ -1,6 +0,0 @@ -from lib2to3.fixer_base import BaseFix - -class FixPreorder(BaseFix): - order = "pre" - - def match(self, node): return False diff --git a/Lib/test/test_lib2to3/data/fixers/no_fixer_cls.py b/Lib/test/test_lib2to3/data/fixers/no_fixer_cls.py deleted file mode 100644 index 506f7949517058..00000000000000 --- a/Lib/test/test_lib2to3/data/fixers/no_fixer_cls.py +++ /dev/null @@ -1 +0,0 @@ -# This is empty so trying to fetch the fixer class gives an AttributeError diff --git a/Lib/test/test_lib2to3/data/fixers/parrot_example.py b/Lib/test/test_lib2to3/data/fixers/parrot_example.py deleted file mode 100644 index 08529280cdb4a1..00000000000000 --- a/Lib/test/test_lib2to3/data/fixers/parrot_example.py +++ /dev/null @@ -1,2 +0,0 @@ -def parrot(): - pass diff --git a/Lib/test/test_lib2to3/data/infinite_recursion.py b/Lib/test/test_lib2to3/data/infinite_recursion.py deleted file mode 100644 index acc62ed4f1a44c..00000000000000 --- a/Lib/test/test_lib2to3/data/infinite_recursion.py +++ /dev/null @@ -1,2669 +0,0 @@ -# Verify that 2to3 falls back from the recursive pattern matching scheme to a -# slower, iterative scheme in the event of a RecursionError. -from ctypes import * -STRING = c_char_p - - -OSUnknownByteOrder = 0 -UIT_PROMPT = 1 -P_PGID = 2 -P_PID = 1 -UIT_ERROR = 5 -UIT_INFO = 4 -UIT_NONE = 0 -P_ALL = 0 -UIT_VERIFY = 2 -OSBigEndian = 2 -UIT_BOOLEAN = 3 -OSLittleEndian = 1 -__darwin_nl_item = c_int -__darwin_wctrans_t = c_int -__darwin_wctype_t = c_ulong -__int8_t = c_byte -__uint8_t = c_ubyte -__int16_t = c_short -__uint16_t = c_ushort -__int32_t = c_int -__uint32_t = c_uint -__int64_t = c_longlong -__uint64_t = c_ulonglong -__darwin_intptr_t = c_long -__darwin_natural_t = c_uint -__darwin_ct_rune_t = c_int -class __mbstate_t(Union): - pass -__mbstate_t._pack_ = 4 -__mbstate_t._fields_ = [ - ('__mbstate8', c_char * 128), - ('_mbstateL', c_longlong), -] -assert sizeof(__mbstate_t) == 128, sizeof(__mbstate_t) -assert alignment(__mbstate_t) == 4, alignment(__mbstate_t) -__darwin_mbstate_t = __mbstate_t -__darwin_ptrdiff_t = c_int -__darwin_size_t = c_ulong -__darwin_va_list = STRING -__darwin_wchar_t = c_int -__darwin_rune_t = __darwin_wchar_t -__darwin_wint_t = c_int -__darwin_clock_t = c_ulong -__darwin_socklen_t = __uint32_t -__darwin_ssize_t = c_long -__darwin_time_t = c_long -sig_atomic_t = c_int -class sigcontext(Structure): - pass -sigcontext._fields_ = [ - ('sc_onstack', c_int), - ('sc_mask', c_int), - ('sc_eax', c_uint), - ('sc_ebx', c_uint), - ('sc_ecx', c_uint), - ('sc_edx', c_uint), - ('sc_edi', c_uint), - ('sc_esi', c_uint), - ('sc_ebp', c_uint), - ('sc_esp', c_uint), - ('sc_ss', c_uint), - ('sc_eflags', c_uint), - ('sc_eip', c_uint), - ('sc_cs', c_uint), - ('sc_ds', c_uint), - ('sc_es', c_uint), - ('sc_fs', c_uint), - ('sc_gs', c_uint), -] -assert sizeof(sigcontext) == 72, sizeof(sigcontext) -assert alignment(sigcontext) == 4, alignment(sigcontext) -u_int8_t = c_ubyte -u_int16_t = c_ushort -u_int32_t = c_uint -u_int64_t = c_ulonglong -int32_t = c_int -register_t = int32_t -user_addr_t = u_int64_t -user_size_t = u_int64_t -int64_t = c_longlong -user_ssize_t = int64_t -user_long_t = int64_t -user_ulong_t = u_int64_t -user_time_t = int64_t -syscall_arg_t = u_int64_t - -# values for unnamed enumeration -class aes_key_st(Structure): - pass -aes_key_st._fields_ = [ - ('rd_key', c_ulong * 60), - ('rounds', c_int), -] -assert sizeof(aes_key_st) == 244, sizeof(aes_key_st) -assert alignment(aes_key_st) == 4, alignment(aes_key_st) -AES_KEY = aes_key_st -class asn1_ctx_st(Structure): - pass -asn1_ctx_st._fields_ = [ - ('p', POINTER(c_ubyte)), - ('eos', c_int), - ('error', c_int), - ('inf', c_int), - ('tag', c_int), - ('xclass', c_int), - ('slen', c_long), - ('max', POINTER(c_ubyte)), - ('q', POINTER(c_ubyte)), - ('pp', POINTER(POINTER(c_ubyte))), - ('line', c_int), -] -assert sizeof(asn1_ctx_st) == 44, sizeof(asn1_ctx_st) -assert alignment(asn1_ctx_st) == 4, alignment(asn1_ctx_st) -ASN1_CTX = asn1_ctx_st -class asn1_object_st(Structure): - pass -asn1_object_st._fields_ = [ - ('sn', STRING), - ('ln', STRING), - ('nid', c_int), - ('length', c_int), - ('data', POINTER(c_ubyte)), - ('flags', c_int), -] -assert sizeof(asn1_object_st) == 24, sizeof(asn1_object_st) -assert alignment(asn1_object_st) == 4, alignment(asn1_object_st) -ASN1_OBJECT = asn1_object_st -class asn1_string_st(Structure): - pass -asn1_string_st._fields_ = [ - ('length', c_int), - ('type', c_int), - ('data', POINTER(c_ubyte)), - ('flags', c_long), -] -assert sizeof(asn1_string_st) == 16, sizeof(asn1_string_st) -assert alignment(asn1_string_st) == 4, alignment(asn1_string_st) -ASN1_STRING = asn1_string_st -class ASN1_ENCODING_st(Structure): - pass -ASN1_ENCODING_st._fields_ = [ - ('enc', POINTER(c_ubyte)), - ('len', c_long), - ('modified', c_int), -] -assert sizeof(ASN1_ENCODING_st) == 12, sizeof(ASN1_ENCODING_st) -assert alignment(ASN1_ENCODING_st) == 4, alignment(ASN1_ENCODING_st) -ASN1_ENCODING = ASN1_ENCODING_st -class asn1_string_table_st(Structure): - pass -asn1_string_table_st._fields_ = [ - ('nid', c_int), - ('minsize', c_long), - ('maxsize', c_long), - ('mask', c_ulong), - ('flags', c_ulong), -] -assert sizeof(asn1_string_table_st) == 20, sizeof(asn1_string_table_st) -assert alignment(asn1_string_table_st) == 4, alignment(asn1_string_table_st) -ASN1_STRING_TABLE = asn1_string_table_st -class ASN1_TEMPLATE_st(Structure): - pass -ASN1_TEMPLATE_st._fields_ = [ -] -ASN1_TEMPLATE = ASN1_TEMPLATE_st -class ASN1_ITEM_st(Structure): - pass -ASN1_ITEM = ASN1_ITEM_st -ASN1_ITEM_st._fields_ = [ -] -class ASN1_TLC_st(Structure): - pass -ASN1_TLC = ASN1_TLC_st -ASN1_TLC_st._fields_ = [ -] -class ASN1_VALUE_st(Structure): - pass -ASN1_VALUE_st._fields_ = [ -] -ASN1_VALUE = ASN1_VALUE_st -ASN1_ITEM_EXP = ASN1_ITEM -class asn1_type_st(Structure): - pass -class N12asn1_type_st4DOLLAR_11E(Union): - pass -ASN1_BOOLEAN = c_int -ASN1_INTEGER = asn1_string_st -ASN1_ENUMERATED = asn1_string_st -ASN1_BIT_STRING = asn1_string_st -ASN1_OCTET_STRING = asn1_string_st -ASN1_PRINTABLESTRING = asn1_string_st -ASN1_T61STRING = asn1_string_st -ASN1_IA5STRING = asn1_string_st -ASN1_GENERALSTRING = asn1_string_st -ASN1_BMPSTRING = asn1_string_st -ASN1_UNIVERSALSTRING = asn1_string_st -ASN1_UTCTIME = asn1_string_st -ASN1_GENERALIZEDTIME = asn1_string_st -ASN1_VISIBLESTRING = asn1_string_st -ASN1_UTF8STRING = asn1_string_st -N12asn1_type_st4DOLLAR_11E._fields_ = [ - ('ptr', STRING), - ('boolean', ASN1_BOOLEAN), - ('asn1_string', POINTER(ASN1_STRING)), - ('object', POINTER(ASN1_OBJECT)), - ('integer', POINTER(ASN1_INTEGER)), - ('enumerated', POINTER(ASN1_ENUMERATED)), - ('bit_string', POINTER(ASN1_BIT_STRING)), - ('octet_string', POINTER(ASN1_OCTET_STRING)), - ('printablestring', POINTER(ASN1_PRINTABLESTRING)), - ('t61string', POINTER(ASN1_T61STRING)), - ('ia5string', POINTER(ASN1_IA5STRING)), - ('generalstring', POINTER(ASN1_GENERALSTRING)), - ('bmpstring', POINTER(ASN1_BMPSTRING)), - ('universalstring', POINTER(ASN1_UNIVERSALSTRING)), - ('utctime', POINTER(ASN1_UTCTIME)), - ('generalizedtime', POINTER(ASN1_GENERALIZEDTIME)), - ('visiblestring', POINTER(ASN1_VISIBLESTRING)), - ('utf8string', POINTER(ASN1_UTF8STRING)), - ('set', POINTER(ASN1_STRING)), - ('sequence', POINTER(ASN1_STRING)), -] -assert sizeof(N12asn1_type_st4DOLLAR_11E) == 4, sizeof(N12asn1_type_st4DOLLAR_11E) -assert alignment(N12asn1_type_st4DOLLAR_11E) == 4, alignment(N12asn1_type_st4DOLLAR_11E) -asn1_type_st._fields_ = [ - ('type', c_int), - ('value', N12asn1_type_st4DOLLAR_11E), -] -assert sizeof(asn1_type_st) == 8, sizeof(asn1_type_st) -assert alignment(asn1_type_st) == 4, alignment(asn1_type_st) -ASN1_TYPE = asn1_type_st -class asn1_method_st(Structure): - pass -asn1_method_st._fields_ = [ - ('i2d', CFUNCTYPE(c_int)), - ('d2i', CFUNCTYPE(STRING)), - ('create', CFUNCTYPE(STRING)), - ('destroy', CFUNCTYPE(None)), -] -assert sizeof(asn1_method_st) == 16, sizeof(asn1_method_st) -assert alignment(asn1_method_st) == 4, alignment(asn1_method_st) -ASN1_METHOD = asn1_method_st -class asn1_header_st(Structure): - pass -asn1_header_st._fields_ = [ - ('header', POINTER(ASN1_OCTET_STRING)), - ('data', STRING), - ('meth', POINTER(ASN1_METHOD)), -] -assert sizeof(asn1_header_st) == 12, sizeof(asn1_header_st) -assert alignment(asn1_header_st) == 4, alignment(asn1_header_st) -ASN1_HEADER = asn1_header_st -class BIT_STRING_BITNAME_st(Structure): - pass -BIT_STRING_BITNAME_st._fields_ = [ - ('bitnum', c_int), - ('lname', STRING), - ('sname', STRING), -] -assert sizeof(BIT_STRING_BITNAME_st) == 12, sizeof(BIT_STRING_BITNAME_st) -assert alignment(BIT_STRING_BITNAME_st) == 4, alignment(BIT_STRING_BITNAME_st) -BIT_STRING_BITNAME = BIT_STRING_BITNAME_st -class bio_st(Structure): - pass -BIO = bio_st -bio_info_cb = CFUNCTYPE(None, POINTER(bio_st), c_int, STRING, c_int, c_long, c_long) -class bio_method_st(Structure): - pass -bio_method_st._fields_ = [ - ('type', c_int), - ('name', STRING), - ('bwrite', CFUNCTYPE(c_int, POINTER(BIO), STRING, c_int)), - ('bread', CFUNCTYPE(c_int, POINTER(BIO), STRING, c_int)), - ('bputs', CFUNCTYPE(c_int, POINTER(BIO), STRING)), - ('bgets', CFUNCTYPE(c_int, POINTER(BIO), STRING, c_int)), - ('ctrl', CFUNCTYPE(c_long, POINTER(BIO), c_int, c_long, c_void_p)), - ('create', CFUNCTYPE(c_int, POINTER(BIO))), - ('destroy', CFUNCTYPE(c_int, POINTER(BIO))), - ('callback_ctrl', CFUNCTYPE(c_long, POINTER(BIO), c_int, POINTER(bio_info_cb))), -] -assert sizeof(bio_method_st) == 40, sizeof(bio_method_st) -assert alignment(bio_method_st) == 4, alignment(bio_method_st) -BIO_METHOD = bio_method_st -class crypto_ex_data_st(Structure): - pass -class stack_st(Structure): - pass -STACK = stack_st -crypto_ex_data_st._fields_ = [ - ('sk', POINTER(STACK)), - ('dummy', c_int), -] -assert sizeof(crypto_ex_data_st) == 8, sizeof(crypto_ex_data_st) -assert alignment(crypto_ex_data_st) == 4, alignment(crypto_ex_data_st) -CRYPTO_EX_DATA = crypto_ex_data_st -bio_st._fields_ = [ - ('method', POINTER(BIO_METHOD)), - ('callback', CFUNCTYPE(c_long, POINTER(bio_st), c_int, STRING, c_int, c_long, c_long)), - ('cb_arg', STRING), - ('init', c_int), - ('shutdown', c_int), - ('flags', c_int), - ('retry_reason', c_int), - ('num', c_int), - ('ptr', c_void_p), - ('next_bio', POINTER(bio_st)), - ('prev_bio', POINTER(bio_st)), - ('references', c_int), - ('num_read', c_ulong), - ('num_write', c_ulong), - ('ex_data', CRYPTO_EX_DATA), -] -assert sizeof(bio_st) == 64, sizeof(bio_st) -assert alignment(bio_st) == 4, alignment(bio_st) -class bio_f_buffer_ctx_struct(Structure): - pass -bio_f_buffer_ctx_struct._fields_ = [ - ('ibuf_size', c_int), - ('obuf_size', c_int), - ('ibuf', STRING), - ('ibuf_len', c_int), - ('ibuf_off', c_int), - ('obuf', STRING), - ('obuf_len', c_int), - ('obuf_off', c_int), -] -assert sizeof(bio_f_buffer_ctx_struct) == 32, sizeof(bio_f_buffer_ctx_struct) -assert alignment(bio_f_buffer_ctx_struct) == 4, alignment(bio_f_buffer_ctx_struct) -BIO_F_BUFFER_CTX = bio_f_buffer_ctx_struct -class hostent(Structure): - pass -hostent._fields_ = [ -] -class bf_key_st(Structure): - pass -bf_key_st._fields_ = [ - ('P', c_uint * 18), - ('S', c_uint * 1024), -] -assert sizeof(bf_key_st) == 4168, sizeof(bf_key_st) -assert alignment(bf_key_st) == 4, alignment(bf_key_st) -BF_KEY = bf_key_st -class bignum_st(Structure): - pass -bignum_st._fields_ = [ - ('d', POINTER(c_ulong)), - ('top', c_int), - ('dmax', c_int), - ('neg', c_int), - ('flags', c_int), -] -assert sizeof(bignum_st) == 20, sizeof(bignum_st) -assert alignment(bignum_st) == 4, alignment(bignum_st) -BIGNUM = bignum_st -class bignum_ctx(Structure): - pass -bignum_ctx._fields_ = [ -] -BN_CTX = bignum_ctx -class bn_blinding_st(Structure): - pass -bn_blinding_st._fields_ = [ - ('init', c_int), - ('A', POINTER(BIGNUM)), - ('Ai', POINTER(BIGNUM)), - ('mod', POINTER(BIGNUM)), - ('thread_id', c_ulong), -] -assert sizeof(bn_blinding_st) == 20, sizeof(bn_blinding_st) -assert alignment(bn_blinding_st) == 4, alignment(bn_blinding_st) -BN_BLINDING = bn_blinding_st -class bn_mont_ctx_st(Structure): - pass -bn_mont_ctx_st._fields_ = [ - ('ri', c_int), - ('RR', BIGNUM), - ('N', BIGNUM), - ('Ni', BIGNUM), - ('n0', c_ulong), - ('flags', c_int), -] -assert sizeof(bn_mont_ctx_st) == 72, sizeof(bn_mont_ctx_st) -assert alignment(bn_mont_ctx_st) == 4, alignment(bn_mont_ctx_st) -BN_MONT_CTX = bn_mont_ctx_st -class bn_recp_ctx_st(Structure): - pass -bn_recp_ctx_st._fields_ = [ - ('N', BIGNUM), - ('Nr', BIGNUM), - ('num_bits', c_int), - ('shift', c_int), - ('flags', c_int), -] -assert sizeof(bn_recp_ctx_st) == 52, sizeof(bn_recp_ctx_st) -assert alignment(bn_recp_ctx_st) == 4, alignment(bn_recp_ctx_st) -BN_RECP_CTX = bn_recp_ctx_st -class buf_mem_st(Structure): - pass -buf_mem_st._fields_ = [ - ('length', c_int), - ('data', STRING), - ('max', c_int), -] -assert sizeof(buf_mem_st) == 12, sizeof(buf_mem_st) -assert alignment(buf_mem_st) == 4, alignment(buf_mem_st) -BUF_MEM = buf_mem_st -class cast_key_st(Structure): - pass -cast_key_st._fields_ = [ - ('data', c_ulong * 32), - ('short_key', c_int), -] -assert sizeof(cast_key_st) == 132, sizeof(cast_key_st) -assert alignment(cast_key_st) == 4, alignment(cast_key_st) -CAST_KEY = cast_key_st -class comp_method_st(Structure): - pass -comp_method_st._fields_ = [ - ('type', c_int), - ('name', STRING), - ('init', CFUNCTYPE(c_int)), - ('finish', CFUNCTYPE(None)), - ('compress', CFUNCTYPE(c_int)), - ('expand', CFUNCTYPE(c_int)), - ('ctrl', CFUNCTYPE(c_long)), - ('callback_ctrl', CFUNCTYPE(c_long)), -] -assert sizeof(comp_method_st) == 32, sizeof(comp_method_st) -assert alignment(comp_method_st) == 4, alignment(comp_method_st) -COMP_METHOD = comp_method_st -class comp_ctx_st(Structure): - pass -comp_ctx_st._fields_ = [ - ('meth', POINTER(COMP_METHOD)), - ('compress_in', c_ulong), - ('compress_out', c_ulong), - ('expand_in', c_ulong), - ('expand_out', c_ulong), - ('ex_data', CRYPTO_EX_DATA), -] -assert sizeof(comp_ctx_st) == 28, sizeof(comp_ctx_st) -assert alignment(comp_ctx_st) == 4, alignment(comp_ctx_st) -COMP_CTX = comp_ctx_st -class CRYPTO_dynlock_value(Structure): - pass -CRYPTO_dynlock_value._fields_ = [ -] -class CRYPTO_dynlock(Structure): - pass -CRYPTO_dynlock._fields_ = [ - ('references', c_int), - ('data', POINTER(CRYPTO_dynlock_value)), -] -assert sizeof(CRYPTO_dynlock) == 8, sizeof(CRYPTO_dynlock) -assert alignment(CRYPTO_dynlock) == 4, alignment(CRYPTO_dynlock) -BIO_dummy = bio_st -CRYPTO_EX_new = CFUNCTYPE(c_int, c_void_p, c_void_p, POINTER(CRYPTO_EX_DATA), c_int, c_long, c_void_p) -CRYPTO_EX_free = CFUNCTYPE(None, c_void_p, c_void_p, POINTER(CRYPTO_EX_DATA), c_int, c_long, c_void_p) -CRYPTO_EX_dup = CFUNCTYPE(c_int, POINTER(CRYPTO_EX_DATA), POINTER(CRYPTO_EX_DATA), c_void_p, c_int, c_long, c_void_p) -class crypto_ex_data_func_st(Structure): - pass -crypto_ex_data_func_st._fields_ = [ - ('argl', c_long), - ('argp', c_void_p), - ('new_func', POINTER(CRYPTO_EX_new)), - ('free_func', POINTER(CRYPTO_EX_free)), - ('dup_func', POINTER(CRYPTO_EX_dup)), -] -assert sizeof(crypto_ex_data_func_st) == 20, sizeof(crypto_ex_data_func_st) -assert alignment(crypto_ex_data_func_st) == 4, alignment(crypto_ex_data_func_st) -CRYPTO_EX_DATA_FUNCS = crypto_ex_data_func_st -class st_CRYPTO_EX_DATA_IMPL(Structure): - pass -CRYPTO_EX_DATA_IMPL = st_CRYPTO_EX_DATA_IMPL -st_CRYPTO_EX_DATA_IMPL._fields_ = [ -] -CRYPTO_MEM_LEAK_CB = CFUNCTYPE(c_void_p, c_ulong, STRING, c_int, c_int, c_void_p) -DES_cblock = c_ubyte * 8 -const_DES_cblock = c_ubyte * 8 -class DES_ks(Structure): - pass -class N6DES_ks3DOLLAR_9E(Union): - pass -N6DES_ks3DOLLAR_9E._fields_ = [ - ('cblock', DES_cblock), - ('deslong', c_ulong * 2), -] -assert sizeof(N6DES_ks3DOLLAR_9E) == 8, sizeof(N6DES_ks3DOLLAR_9E) -assert alignment(N6DES_ks3DOLLAR_9E) == 4, alignment(N6DES_ks3DOLLAR_9E) -DES_ks._fields_ = [ - ('ks', N6DES_ks3DOLLAR_9E * 16), -] -assert sizeof(DES_ks) == 128, sizeof(DES_ks) -assert alignment(DES_ks) == 4, alignment(DES_ks) -DES_key_schedule = DES_ks -_ossl_old_des_cblock = c_ubyte * 8 -class _ossl_old_des_ks_struct(Structure): - pass -class N23_ossl_old_des_ks_struct4DOLLAR_10E(Union): - pass -N23_ossl_old_des_ks_struct4DOLLAR_10E._fields_ = [ - ('_', _ossl_old_des_cblock), - ('pad', c_ulong * 2), -] -assert sizeof(N23_ossl_old_des_ks_struct4DOLLAR_10E) == 8, sizeof(N23_ossl_old_des_ks_struct4DOLLAR_10E) -assert alignment(N23_ossl_old_des_ks_struct4DOLLAR_10E) == 4, alignment(N23_ossl_old_des_ks_struct4DOLLAR_10E) -_ossl_old_des_ks_struct._fields_ = [ - ('ks', N23_ossl_old_des_ks_struct4DOLLAR_10E), -] -assert sizeof(_ossl_old_des_ks_struct) == 8, sizeof(_ossl_old_des_ks_struct) -assert alignment(_ossl_old_des_ks_struct) == 4, alignment(_ossl_old_des_ks_struct) -_ossl_old_des_key_schedule = _ossl_old_des_ks_struct * 16 -class dh_st(Structure): - pass -DH = dh_st -class dh_method(Structure): - pass -dh_method._fields_ = [ - ('name', STRING), - ('generate_key', CFUNCTYPE(c_int, POINTER(DH))), - ('compute_key', CFUNCTYPE(c_int, POINTER(c_ubyte), POINTER(BIGNUM), POINTER(DH))), - ('bn_mod_exp', CFUNCTYPE(c_int, POINTER(DH), POINTER(BIGNUM), POINTER(BIGNUM), POINTER(BIGNUM), POINTER(BIGNUM), POINTER(BN_CTX), POINTER(BN_MONT_CTX))), - ('init', CFUNCTYPE(c_int, POINTER(DH))), - ('finish', CFUNCTYPE(c_int, POINTER(DH))), - ('flags', c_int), - ('app_data', STRING), -] -assert sizeof(dh_method) == 32, sizeof(dh_method) -assert alignment(dh_method) == 4, alignment(dh_method) -DH_METHOD = dh_method -class engine_st(Structure): - pass -ENGINE = engine_st -dh_st._fields_ = [ - ('pad', c_int), - ('version', c_int), - ('p', POINTER(BIGNUM)), - ('g', POINTER(BIGNUM)), - ('length', c_long), - ('pub_key', POINTER(BIGNUM)), - ('priv_key', POINTER(BIGNUM)), - ('flags', c_int), - ('method_mont_p', STRING), - ('q', POINTER(BIGNUM)), - ('j', POINTER(BIGNUM)), - ('seed', POINTER(c_ubyte)), - ('seedlen', c_int), - ('counter', POINTER(BIGNUM)), - ('references', c_int), - ('ex_data', CRYPTO_EX_DATA), - ('meth', POINTER(DH_METHOD)), - ('engine', POINTER(ENGINE)), -] -assert sizeof(dh_st) == 76, sizeof(dh_st) -assert alignment(dh_st) == 4, alignment(dh_st) -class dsa_st(Structure): - pass -DSA = dsa_st -class DSA_SIG_st(Structure): - pass -DSA_SIG_st._fields_ = [ - ('r', POINTER(BIGNUM)), - ('s', POINTER(BIGNUM)), -] -assert sizeof(DSA_SIG_st) == 8, sizeof(DSA_SIG_st) -assert alignment(DSA_SIG_st) == 4, alignment(DSA_SIG_st) -DSA_SIG = DSA_SIG_st -class dsa_method(Structure): - pass -dsa_method._fields_ = [ - ('name', STRING), - ('dsa_do_sign', CFUNCTYPE(POINTER(DSA_SIG), POINTER(c_ubyte), c_int, POINTER(DSA))), - ('dsa_sign_setup', CFUNCTYPE(c_int, POINTER(DSA), POINTER(BN_CTX), POINTER(POINTER(BIGNUM)), POINTER(POINTER(BIGNUM)))), - ('dsa_do_verify', CFUNCTYPE(c_int, POINTER(c_ubyte), c_int, POINTER(DSA_SIG), POINTER(DSA))), - ('dsa_mod_exp', CFUNCTYPE(c_int, POINTER(DSA), POINTER(BIGNUM), POINTER(BIGNUM), POINTER(BIGNUM), POINTER(BIGNUM), POINTER(BIGNUM), POINTER(BIGNUM), POINTER(BN_CTX), POINTER(BN_MONT_CTX))), - ('bn_mod_exp', CFUNCTYPE(c_int, POINTER(DSA), POINTER(BIGNUM), POINTER(BIGNUM), POINTER(BIGNUM), POINTER(BIGNUM), POINTER(BN_CTX), POINTER(BN_MONT_CTX))), - ('init', CFUNCTYPE(c_int, POINTER(DSA))), - ('finish', CFUNCTYPE(c_int, POINTER(DSA))), - ('flags', c_int), - ('app_data', STRING), -] -assert sizeof(dsa_method) == 40, sizeof(dsa_method) -assert alignment(dsa_method) == 4, alignment(dsa_method) -DSA_METHOD = dsa_method -dsa_st._fields_ = [ - ('pad', c_int), - ('version', c_long), - ('write_params', c_int), - ('p', POINTER(BIGNUM)), - ('q', POINTER(BIGNUM)), - ('g', POINTER(BIGNUM)), - ('pub_key', POINTER(BIGNUM)), - ('priv_key', POINTER(BIGNUM)), - ('kinv', POINTER(BIGNUM)), - ('r', POINTER(BIGNUM)), - ('flags', c_int), - ('method_mont_p', STRING), - ('references', c_int), - ('ex_data', CRYPTO_EX_DATA), - ('meth', POINTER(DSA_METHOD)), - ('engine', POINTER(ENGINE)), -] -assert sizeof(dsa_st) == 68, sizeof(dsa_st) -assert alignment(dsa_st) == 4, alignment(dsa_st) -class evp_pkey_st(Structure): - pass -class N11evp_pkey_st4DOLLAR_12E(Union): - pass -class rsa_st(Structure): - pass -N11evp_pkey_st4DOLLAR_12E._fields_ = [ - ('ptr', STRING), - ('rsa', POINTER(rsa_st)), - ('dsa', POINTER(dsa_st)), - ('dh', POINTER(dh_st)), -] -assert sizeof(N11evp_pkey_st4DOLLAR_12E) == 4, sizeof(N11evp_pkey_st4DOLLAR_12E) -assert alignment(N11evp_pkey_st4DOLLAR_12E) == 4, alignment(N11evp_pkey_st4DOLLAR_12E) -evp_pkey_st._fields_ = [ - ('type', c_int), - ('save_type', c_int), - ('references', c_int), - ('pkey', N11evp_pkey_st4DOLLAR_12E), - ('save_parameters', c_int), - ('attributes', POINTER(STACK)), -] -assert sizeof(evp_pkey_st) == 24, sizeof(evp_pkey_st) -assert alignment(evp_pkey_st) == 4, alignment(evp_pkey_st) -class env_md_st(Structure): - pass -class env_md_ctx_st(Structure): - pass -EVP_MD_CTX = env_md_ctx_st -env_md_st._fields_ = [ - ('type', c_int), - ('pkey_type', c_int), - ('md_size', c_int), - ('flags', c_ulong), - ('init', CFUNCTYPE(c_int, POINTER(EVP_MD_CTX))), - ('update', CFUNCTYPE(c_int, POINTER(EVP_MD_CTX), c_void_p, c_ulong)), - ('final', CFUNCTYPE(c_int, POINTER(EVP_MD_CTX), POINTER(c_ubyte))), - ('copy', CFUNCTYPE(c_int, POINTER(EVP_MD_CTX), POINTER(EVP_MD_CTX))), - ('cleanup', CFUNCTYPE(c_int, POINTER(EVP_MD_CTX))), - ('sign', CFUNCTYPE(c_int)), - ('verify', CFUNCTYPE(c_int)), - ('required_pkey_type', c_int * 5), - ('block_size', c_int), - ('ctx_size', c_int), -] -assert sizeof(env_md_st) == 72, sizeof(env_md_st) -assert alignment(env_md_st) == 4, alignment(env_md_st) -EVP_MD = env_md_st -env_md_ctx_st._fields_ = [ - ('digest', POINTER(EVP_MD)), - ('engine', POINTER(ENGINE)), - ('flags', c_ulong), - ('md_data', c_void_p), -] -assert sizeof(env_md_ctx_st) == 16, sizeof(env_md_ctx_st) -assert alignment(env_md_ctx_st) == 4, alignment(env_md_ctx_st) -class evp_cipher_st(Structure): - pass -class evp_cipher_ctx_st(Structure): - pass -EVP_CIPHER_CTX = evp_cipher_ctx_st -evp_cipher_st._fields_ = [ - ('nid', c_int), - ('block_size', c_int), - ('key_len', c_int), - ('iv_len', c_int), - ('flags', c_ulong), - ('init', CFUNCTYPE(c_int, POINTER(EVP_CIPHER_CTX), POINTER(c_ubyte), POINTER(c_ubyte), c_int)), - ('do_cipher', CFUNCTYPE(c_int, POINTER(EVP_CIPHER_CTX), POINTER(c_ubyte), POINTER(c_ubyte), c_uint)), - ('cleanup', CFUNCTYPE(c_int, POINTER(EVP_CIPHER_CTX))), - ('ctx_size', c_int), - ('set_asn1_parameters', CFUNCTYPE(c_int, POINTER(EVP_CIPHER_CTX), POINTER(ASN1_TYPE))), - ('get_asn1_parameters', CFUNCTYPE(c_int, POINTER(EVP_CIPHER_CTX), POINTER(ASN1_TYPE))), - ('ctrl', CFUNCTYPE(c_int, POINTER(EVP_CIPHER_CTX), c_int, c_int, c_void_p)), - ('app_data', c_void_p), -] -assert sizeof(evp_cipher_st) == 52, sizeof(evp_cipher_st) -assert alignment(evp_cipher_st) == 4, alignment(evp_cipher_st) -class evp_cipher_info_st(Structure): - pass -EVP_CIPHER = evp_cipher_st -evp_cipher_info_st._fields_ = [ - ('cipher', POINTER(EVP_CIPHER)), - ('iv', c_ubyte * 16), -] -assert sizeof(evp_cipher_info_st) == 20, sizeof(evp_cipher_info_st) -assert alignment(evp_cipher_info_st) == 4, alignment(evp_cipher_info_st) -EVP_CIPHER_INFO = evp_cipher_info_st -evp_cipher_ctx_st._fields_ = [ - ('cipher', POINTER(EVP_CIPHER)), - ('engine', POINTER(ENGINE)), - ('encrypt', c_int), - ('buf_len', c_int), - ('oiv', c_ubyte * 16), - ('iv', c_ubyte * 16), - ('buf', c_ubyte * 32), - ('num', c_int), - ('app_data', c_void_p), - ('key_len', c_int), - ('flags', c_ulong), - ('cipher_data', c_void_p), - ('final_used', c_int), - ('block_mask', c_int), - ('final', c_ubyte * 32), -] -assert sizeof(evp_cipher_ctx_st) == 140, sizeof(evp_cipher_ctx_st) -assert alignment(evp_cipher_ctx_st) == 4, alignment(evp_cipher_ctx_st) -class evp_Encode_Ctx_st(Structure): - pass -evp_Encode_Ctx_st._fields_ = [ - ('num', c_int), - ('length', c_int), - ('enc_data', c_ubyte * 80), - ('line_num', c_int), - ('expect_nl', c_int), -] -assert sizeof(evp_Encode_Ctx_st) == 96, sizeof(evp_Encode_Ctx_st) -assert alignment(evp_Encode_Ctx_st) == 4, alignment(evp_Encode_Ctx_st) -EVP_ENCODE_CTX = evp_Encode_Ctx_st -EVP_PBE_KEYGEN = CFUNCTYPE(c_int, POINTER(EVP_CIPHER_CTX), STRING, c_int, POINTER(ASN1_TYPE), POINTER(EVP_CIPHER), POINTER(EVP_MD), c_int) -class lhash_node_st(Structure): - pass -lhash_node_st._fields_ = [ - ('data', c_void_p), - ('next', POINTER(lhash_node_st)), - ('hash', c_ulong), -] -assert sizeof(lhash_node_st) == 12, sizeof(lhash_node_st) -assert alignment(lhash_node_st) == 4, alignment(lhash_node_st) -LHASH_NODE = lhash_node_st -LHASH_COMP_FN_TYPE = CFUNCTYPE(c_int, c_void_p, c_void_p) -LHASH_HASH_FN_TYPE = CFUNCTYPE(c_ulong, c_void_p) -LHASH_DOALL_FN_TYPE = CFUNCTYPE(None, c_void_p) -LHASH_DOALL_ARG_FN_TYPE = CFUNCTYPE(None, c_void_p, c_void_p) -class lhash_st(Structure): - pass -lhash_st._fields_ = [ - ('b', POINTER(POINTER(LHASH_NODE))), - ('comp', LHASH_COMP_FN_TYPE), - ('hash', LHASH_HASH_FN_TYPE), - ('num_nodes', c_uint), - ('num_alloc_nodes', c_uint), - ('p', c_uint), - ('pmax', c_uint), - ('up_load', c_ulong), - ('down_load', c_ulong), - ('num_items', c_ulong), - ('num_expands', c_ulong), - ('num_expand_reallocs', c_ulong), - ('num_contracts', c_ulong), - ('num_contract_reallocs', c_ulong), - ('num_hash_calls', c_ulong), - ('num_comp_calls', c_ulong), - ('num_insert', c_ulong), - ('num_replace', c_ulong), - ('num_delete', c_ulong), - ('num_no_delete', c_ulong), - ('num_retrieve', c_ulong), - ('num_retrieve_miss', c_ulong), - ('num_hash_comps', c_ulong), - ('error', c_int), -] -assert sizeof(lhash_st) == 96, sizeof(lhash_st) -assert alignment(lhash_st) == 4, alignment(lhash_st) -LHASH = lhash_st -class MD2state_st(Structure): - pass -MD2state_st._fields_ = [ - ('num', c_int), - ('data', c_ubyte * 16), - ('cksm', c_uint * 16), - ('state', c_uint * 16), -] -assert sizeof(MD2state_st) == 148, sizeof(MD2state_st) -assert alignment(MD2state_st) == 4, alignment(MD2state_st) -MD2_CTX = MD2state_st -class MD4state_st(Structure): - pass -MD4state_st._fields_ = [ - ('A', c_uint), - ('B', c_uint), - ('C', c_uint), - ('D', c_uint), - ('Nl', c_uint), - ('Nh', c_uint), - ('data', c_uint * 16), - ('num', c_int), -] -assert sizeof(MD4state_st) == 92, sizeof(MD4state_st) -assert alignment(MD4state_st) == 4, alignment(MD4state_st) -MD4_CTX = MD4state_st -class MD5state_st(Structure): - pass -MD5state_st._fields_ = [ - ('A', c_uint), - ('B', c_uint), - ('C', c_uint), - ('D', c_uint), - ('Nl', c_uint), - ('Nh', c_uint), - ('data', c_uint * 16), - ('num', c_int), -] -assert sizeof(MD5state_st) == 92, sizeof(MD5state_st) -assert alignment(MD5state_st) == 4, alignment(MD5state_st) -MD5_CTX = MD5state_st -class mdc2_ctx_st(Structure): - pass -mdc2_ctx_st._fields_ = [ - ('num', c_int), - ('data', c_ubyte * 8), - ('h', DES_cblock), - ('hh', DES_cblock), - ('pad_type', c_int), -] -assert sizeof(mdc2_ctx_st) == 32, sizeof(mdc2_ctx_st) -assert alignment(mdc2_ctx_st) == 4, alignment(mdc2_ctx_st) -MDC2_CTX = mdc2_ctx_st -class obj_name_st(Structure): - pass -obj_name_st._fields_ = [ - ('type', c_int), - ('alias', c_int), - ('name', STRING), - ('data', STRING), -] -assert sizeof(obj_name_st) == 16, sizeof(obj_name_st) -assert alignment(obj_name_st) == 4, alignment(obj_name_st) -OBJ_NAME = obj_name_st -ASN1_TIME = asn1_string_st -ASN1_NULL = c_int -EVP_PKEY = evp_pkey_st -class x509_st(Structure): - pass -X509 = x509_st -class X509_algor_st(Structure): - pass -X509_ALGOR = X509_algor_st -class X509_crl_st(Structure): - pass -X509_CRL = X509_crl_st -class X509_name_st(Structure): - pass -X509_NAME = X509_name_st -class x509_store_st(Structure): - pass -X509_STORE = x509_store_st -class x509_store_ctx_st(Structure): - pass -X509_STORE_CTX = x509_store_ctx_st -engine_st._fields_ = [ -] -class PEM_Encode_Seal_st(Structure): - pass -PEM_Encode_Seal_st._fields_ = [ - ('encode', EVP_ENCODE_CTX), - ('md', EVP_MD_CTX), - ('cipher', EVP_CIPHER_CTX), -] -assert sizeof(PEM_Encode_Seal_st) == 252, sizeof(PEM_Encode_Seal_st) -assert alignment(PEM_Encode_Seal_st) == 4, alignment(PEM_Encode_Seal_st) -PEM_ENCODE_SEAL_CTX = PEM_Encode_Seal_st -class pem_recip_st(Structure): - pass -pem_recip_st._fields_ = [ - ('name', STRING), - ('dn', POINTER(X509_NAME)), - ('cipher', c_int), - ('key_enc', c_int), -] -assert sizeof(pem_recip_st) == 16, sizeof(pem_recip_st) -assert alignment(pem_recip_st) == 4, alignment(pem_recip_st) -PEM_USER = pem_recip_st -class pem_ctx_st(Structure): - pass -class N10pem_ctx_st4DOLLAR_16E(Structure): - pass -N10pem_ctx_st4DOLLAR_16E._fields_ = [ - ('version', c_int), - ('mode', c_int), -] -assert sizeof(N10pem_ctx_st4DOLLAR_16E) == 8, sizeof(N10pem_ctx_st4DOLLAR_16E) -assert alignment(N10pem_ctx_st4DOLLAR_16E) == 4, alignment(N10pem_ctx_st4DOLLAR_16E) -class N10pem_ctx_st4DOLLAR_17E(Structure): - pass -N10pem_ctx_st4DOLLAR_17E._fields_ = [ - ('cipher', c_int), -] -assert sizeof(N10pem_ctx_st4DOLLAR_17E) == 4, sizeof(N10pem_ctx_st4DOLLAR_17E) -assert alignment(N10pem_ctx_st4DOLLAR_17E) == 4, alignment(N10pem_ctx_st4DOLLAR_17E) -pem_ctx_st._fields_ = [ - ('type', c_int), - ('proc_type', N10pem_ctx_st4DOLLAR_16E), - ('domain', STRING), - ('DEK_info', N10pem_ctx_st4DOLLAR_17E), - ('originator', POINTER(PEM_USER)), - ('num_recipient', c_int), - ('recipient', POINTER(POINTER(PEM_USER))), - ('x509_chain', POINTER(STACK)), - ('md', POINTER(EVP_MD)), - ('md_enc', c_int), - ('md_len', c_int), - ('md_data', STRING), - ('dec', POINTER(EVP_CIPHER)), - ('key_len', c_int), - ('key', POINTER(c_ubyte)), - ('data_enc', c_int), - ('data_len', c_int), - ('data', POINTER(c_ubyte)), -] -assert sizeof(pem_ctx_st) == 76, sizeof(pem_ctx_st) -assert alignment(pem_ctx_st) == 4, alignment(pem_ctx_st) -PEM_CTX = pem_ctx_st -pem_password_cb = CFUNCTYPE(c_int, STRING, c_int, c_int, c_void_p) -class pkcs7_issuer_and_serial_st(Structure): - pass -pkcs7_issuer_and_serial_st._fields_ = [ - ('issuer', POINTER(X509_NAME)), - ('serial', POINTER(ASN1_INTEGER)), -] -assert sizeof(pkcs7_issuer_and_serial_st) == 8, sizeof(pkcs7_issuer_and_serial_st) -assert alignment(pkcs7_issuer_and_serial_st) == 4, alignment(pkcs7_issuer_and_serial_st) -PKCS7_ISSUER_AND_SERIAL = pkcs7_issuer_and_serial_st -class pkcs7_signer_info_st(Structure): - pass -pkcs7_signer_info_st._fields_ = [ - ('version', POINTER(ASN1_INTEGER)), - ('issuer_and_serial', POINTER(PKCS7_ISSUER_AND_SERIAL)), - ('digest_alg', POINTER(X509_ALGOR)), - ('auth_attr', POINTER(STACK)), - ('digest_enc_alg', POINTER(X509_ALGOR)), - ('enc_digest', POINTER(ASN1_OCTET_STRING)), - ('unauth_attr', POINTER(STACK)), - ('pkey', POINTER(EVP_PKEY)), -] -assert sizeof(pkcs7_signer_info_st) == 32, sizeof(pkcs7_signer_info_st) -assert alignment(pkcs7_signer_info_st) == 4, alignment(pkcs7_signer_info_st) -PKCS7_SIGNER_INFO = pkcs7_signer_info_st -class pkcs7_recip_info_st(Structure): - pass -pkcs7_recip_info_st._fields_ = [ - ('version', POINTER(ASN1_INTEGER)), - ('issuer_and_serial', POINTER(PKCS7_ISSUER_AND_SERIAL)), - ('key_enc_algor', POINTER(X509_ALGOR)), - ('enc_key', POINTER(ASN1_OCTET_STRING)), - ('cert', POINTER(X509)), -] -assert sizeof(pkcs7_recip_info_st) == 20, sizeof(pkcs7_recip_info_st) -assert alignment(pkcs7_recip_info_st) == 4, alignment(pkcs7_recip_info_st) -PKCS7_RECIP_INFO = pkcs7_recip_info_st -class pkcs7_signed_st(Structure): - pass -class pkcs7_st(Structure): - pass -pkcs7_signed_st._fields_ = [ - ('version', POINTER(ASN1_INTEGER)), - ('md_algs', POINTER(STACK)), - ('cert', POINTER(STACK)), - ('crl', POINTER(STACK)), - ('signer_info', POINTER(STACK)), - ('contents', POINTER(pkcs7_st)), -] -assert sizeof(pkcs7_signed_st) == 24, sizeof(pkcs7_signed_st) -assert alignment(pkcs7_signed_st) == 4, alignment(pkcs7_signed_st) -PKCS7_SIGNED = pkcs7_signed_st -class pkcs7_enc_content_st(Structure): - pass -pkcs7_enc_content_st._fields_ = [ - ('content_type', POINTER(ASN1_OBJECT)), - ('algorithm', POINTER(X509_ALGOR)), - ('enc_data', POINTER(ASN1_OCTET_STRING)), - ('cipher', POINTER(EVP_CIPHER)), -] -assert sizeof(pkcs7_enc_content_st) == 16, sizeof(pkcs7_enc_content_st) -assert alignment(pkcs7_enc_content_st) == 4, alignment(pkcs7_enc_content_st) -PKCS7_ENC_CONTENT = pkcs7_enc_content_st -class pkcs7_enveloped_st(Structure): - pass -pkcs7_enveloped_st._fields_ = [ - ('version', POINTER(ASN1_INTEGER)), - ('recipientinfo', POINTER(STACK)), - ('enc_data', POINTER(PKCS7_ENC_CONTENT)), -] -assert sizeof(pkcs7_enveloped_st) == 12, sizeof(pkcs7_enveloped_st) -assert alignment(pkcs7_enveloped_st) == 4, alignment(pkcs7_enveloped_st) -PKCS7_ENVELOPE = pkcs7_enveloped_st -class pkcs7_signedandenveloped_st(Structure): - pass -pkcs7_signedandenveloped_st._fields_ = [ - ('version', POINTER(ASN1_INTEGER)), - ('md_algs', POINTER(STACK)), - ('cert', POINTER(STACK)), - ('crl', POINTER(STACK)), - ('signer_info', POINTER(STACK)), - ('enc_data', POINTER(PKCS7_ENC_CONTENT)), - ('recipientinfo', POINTER(STACK)), -] -assert sizeof(pkcs7_signedandenveloped_st) == 28, sizeof(pkcs7_signedandenveloped_st) -assert alignment(pkcs7_signedandenveloped_st) == 4, alignment(pkcs7_signedandenveloped_st) -PKCS7_SIGN_ENVELOPE = pkcs7_signedandenveloped_st -class pkcs7_digest_st(Structure): - pass -pkcs7_digest_st._fields_ = [ - ('version', POINTER(ASN1_INTEGER)), - ('md', POINTER(X509_ALGOR)), - ('contents', POINTER(pkcs7_st)), - ('digest', POINTER(ASN1_OCTET_STRING)), -] -assert sizeof(pkcs7_digest_st) == 16, sizeof(pkcs7_digest_st) -assert alignment(pkcs7_digest_st) == 4, alignment(pkcs7_digest_st) -PKCS7_DIGEST = pkcs7_digest_st -class pkcs7_encrypted_st(Structure): - pass -pkcs7_encrypted_st._fields_ = [ - ('version', POINTER(ASN1_INTEGER)), - ('enc_data', POINTER(PKCS7_ENC_CONTENT)), -] -assert sizeof(pkcs7_encrypted_st) == 8, sizeof(pkcs7_encrypted_st) -assert alignment(pkcs7_encrypted_st) == 4, alignment(pkcs7_encrypted_st) -PKCS7_ENCRYPT = pkcs7_encrypted_st -class N8pkcs7_st4DOLLAR_15E(Union): - pass -N8pkcs7_st4DOLLAR_15E._fields_ = [ - ('ptr', STRING), - ('data', POINTER(ASN1_OCTET_STRING)), - ('sign', POINTER(PKCS7_SIGNED)), - ('enveloped', POINTER(PKCS7_ENVELOPE)), - ('signed_and_enveloped', POINTER(PKCS7_SIGN_ENVELOPE)), - ('digest', POINTER(PKCS7_DIGEST)), - ('encrypted', POINTER(PKCS7_ENCRYPT)), - ('other', POINTER(ASN1_TYPE)), -] -assert sizeof(N8pkcs7_st4DOLLAR_15E) == 4, sizeof(N8pkcs7_st4DOLLAR_15E) -assert alignment(N8pkcs7_st4DOLLAR_15E) == 4, alignment(N8pkcs7_st4DOLLAR_15E) -pkcs7_st._fields_ = [ - ('asn1', POINTER(c_ubyte)), - ('length', c_long), - ('state', c_int), - ('detached', c_int), - ('type', POINTER(ASN1_OBJECT)), - ('d', N8pkcs7_st4DOLLAR_15E), -] -assert sizeof(pkcs7_st) == 24, sizeof(pkcs7_st) -assert alignment(pkcs7_st) == 4, alignment(pkcs7_st) -PKCS7 = pkcs7_st -class rc2_key_st(Structure): - pass -rc2_key_st._fields_ = [ - ('data', c_uint * 64), -] -assert sizeof(rc2_key_st) == 256, sizeof(rc2_key_st) -assert alignment(rc2_key_st) == 4, alignment(rc2_key_st) -RC2_KEY = rc2_key_st -class rc4_key_st(Structure): - pass -rc4_key_st._fields_ = [ - ('x', c_ubyte), - ('y', c_ubyte), - ('data', c_ubyte * 256), -] -assert sizeof(rc4_key_st) == 258, sizeof(rc4_key_st) -assert alignment(rc4_key_st) == 1, alignment(rc4_key_st) -RC4_KEY = rc4_key_st -class rc5_key_st(Structure): - pass -rc5_key_st._fields_ = [ - ('rounds', c_int), - ('data', c_ulong * 34), -] -assert sizeof(rc5_key_st) == 140, sizeof(rc5_key_st) -assert alignment(rc5_key_st) == 4, alignment(rc5_key_st) -RC5_32_KEY = rc5_key_st -class RIPEMD160state_st(Structure): - pass -RIPEMD160state_st._fields_ = [ - ('A', c_uint), - ('B', c_uint), - ('C', c_uint), - ('D', c_uint), - ('E', c_uint), - ('Nl', c_uint), - ('Nh', c_uint), - ('data', c_uint * 16), - ('num', c_int), -] -assert sizeof(RIPEMD160state_st) == 96, sizeof(RIPEMD160state_st) -assert alignment(RIPEMD160state_st) == 4, alignment(RIPEMD160state_st) -RIPEMD160_CTX = RIPEMD160state_st -RSA = rsa_st -class rsa_meth_st(Structure): - pass -rsa_meth_st._fields_ = [ - ('name', STRING), - ('rsa_pub_enc', CFUNCTYPE(c_int, c_int, POINTER(c_ubyte), POINTER(c_ubyte), POINTER(RSA), c_int)), - ('rsa_pub_dec', CFUNCTYPE(c_int, c_int, POINTER(c_ubyte), POINTER(c_ubyte), POINTER(RSA), c_int)), - ('rsa_priv_enc', CFUNCTYPE(c_int, c_int, POINTER(c_ubyte), POINTER(c_ubyte), POINTER(RSA), c_int)), - ('rsa_priv_dec', CFUNCTYPE(c_int, c_int, POINTER(c_ubyte), POINTER(c_ubyte), POINTER(RSA), c_int)), - ('rsa_mod_exp', CFUNCTYPE(c_int, POINTER(BIGNUM), POINTER(BIGNUM), POINTER(RSA))), - ('bn_mod_exp', CFUNCTYPE(c_int, POINTER(BIGNUM), POINTER(BIGNUM), POINTER(BIGNUM), POINTER(BIGNUM), POINTER(BN_CTX), POINTER(BN_MONT_CTX))), - ('init', CFUNCTYPE(c_int, POINTER(RSA))), - ('finish', CFUNCTYPE(c_int, POINTER(RSA))), - ('flags', c_int), - ('app_data', STRING), - ('rsa_sign', CFUNCTYPE(c_int, c_int, POINTER(c_ubyte), c_uint, POINTER(c_ubyte), POINTER(c_uint), POINTER(RSA))), - ('rsa_verify', CFUNCTYPE(c_int, c_int, POINTER(c_ubyte), c_uint, POINTER(c_ubyte), c_uint, POINTER(RSA))), -] -assert sizeof(rsa_meth_st) == 52, sizeof(rsa_meth_st) -assert alignment(rsa_meth_st) == 4, alignment(rsa_meth_st) -RSA_METHOD = rsa_meth_st -rsa_st._fields_ = [ - ('pad', c_int), - ('version', c_long), - ('meth', POINTER(RSA_METHOD)), - ('engine', POINTER(ENGINE)), - ('n', POINTER(BIGNUM)), - ('e', POINTER(BIGNUM)), - ('d', POINTER(BIGNUM)), - ('p', POINTER(BIGNUM)), - ('q', POINTER(BIGNUM)), - ('dmp1', POINTER(BIGNUM)), - ('dmq1', POINTER(BIGNUM)), - ('iqmp', POINTER(BIGNUM)), - ('ex_data', CRYPTO_EX_DATA), - ('references', c_int), - ('flags', c_int), - ('_method_mod_n', POINTER(BN_MONT_CTX)), - ('_method_mod_p', POINTER(BN_MONT_CTX)), - ('_method_mod_q', POINTER(BN_MONT_CTX)), - ('bignum_data', STRING), - ('blinding', POINTER(BN_BLINDING)), -] -assert sizeof(rsa_st) == 84, sizeof(rsa_st) -assert alignment(rsa_st) == 4, alignment(rsa_st) -openssl_fptr = CFUNCTYPE(None) -class SHAstate_st(Structure): - pass -SHAstate_st._fields_ = [ - ('h0', c_uint), - ('h1', c_uint), - ('h2', c_uint), - ('h3', c_uint), - ('h4', c_uint), - ('Nl', c_uint), - ('Nh', c_uint), - ('data', c_uint * 16), - ('num', c_int), -] -assert sizeof(SHAstate_st) == 96, sizeof(SHAstate_st) -assert alignment(SHAstate_st) == 4, alignment(SHAstate_st) -SHA_CTX = SHAstate_st -class ssl_st(Structure): - pass -ssl_crock_st = POINTER(ssl_st) -class ssl_cipher_st(Structure): - pass -ssl_cipher_st._fields_ = [ - ('valid', c_int), - ('name', STRING), - ('id', c_ulong), - ('algorithms', c_ulong), - ('algo_strength', c_ulong), - ('algorithm2', c_ulong), - ('strength_bits', c_int), - ('alg_bits', c_int), - ('mask', c_ulong), - ('mask_strength', c_ulong), -] -assert sizeof(ssl_cipher_st) == 40, sizeof(ssl_cipher_st) -assert alignment(ssl_cipher_st) == 4, alignment(ssl_cipher_st) -SSL_CIPHER = ssl_cipher_st -SSL = ssl_st -class ssl_ctx_st(Structure): - pass -SSL_CTX = ssl_ctx_st -class ssl_method_st(Structure): - pass -class ssl3_enc_method(Structure): - pass -ssl_method_st._fields_ = [ - ('version', c_int), - ('ssl_new', CFUNCTYPE(c_int, POINTER(SSL))), - ('ssl_clear', CFUNCTYPE(None, POINTER(SSL))), - ('ssl_free', CFUNCTYPE(None, POINTER(SSL))), - ('ssl_accept', CFUNCTYPE(c_int, POINTER(SSL))), - ('ssl_connect', CFUNCTYPE(c_int, POINTER(SSL))), - ('ssl_read', CFUNCTYPE(c_int, POINTER(SSL), c_void_p, c_int)), - ('ssl_peek', CFUNCTYPE(c_int, POINTER(SSL), c_void_p, c_int)), - ('ssl_write', CFUNCTYPE(c_int, POINTER(SSL), c_void_p, c_int)), - ('ssl_shutdown', CFUNCTYPE(c_int, POINTER(SSL))), - ('ssl_renegotiate', CFUNCTYPE(c_int, POINTER(SSL))), - ('ssl_renegotiate_check', CFUNCTYPE(c_int, POINTER(SSL))), - ('ssl_ctrl', CFUNCTYPE(c_long, POINTER(SSL), c_int, c_long, c_void_p)), - ('ssl_ctx_ctrl', CFUNCTYPE(c_long, POINTER(SSL_CTX), c_int, c_long, c_void_p)), - ('get_cipher_by_char', CFUNCTYPE(POINTER(SSL_CIPHER), POINTER(c_ubyte))), - ('put_cipher_by_char', CFUNCTYPE(c_int, POINTER(SSL_CIPHER), POINTER(c_ubyte))), - ('ssl_pending', CFUNCTYPE(c_int, POINTER(SSL))), - ('num_ciphers', CFUNCTYPE(c_int)), - ('get_cipher', CFUNCTYPE(POINTER(SSL_CIPHER), c_uint)), - ('get_ssl_method', CFUNCTYPE(POINTER(ssl_method_st), c_int)), - ('get_timeout', CFUNCTYPE(c_long)), - ('ssl3_enc', POINTER(ssl3_enc_method)), - ('ssl_version', CFUNCTYPE(c_int)), - ('ssl_callback_ctrl', CFUNCTYPE(c_long, POINTER(SSL), c_int, CFUNCTYPE(None))), - ('ssl_ctx_callback_ctrl', CFUNCTYPE(c_long, POINTER(SSL_CTX), c_int, CFUNCTYPE(None))), -] -assert sizeof(ssl_method_st) == 100, sizeof(ssl_method_st) -assert alignment(ssl_method_st) == 4, alignment(ssl_method_st) -ssl3_enc_method._fields_ = [ -] -SSL_METHOD = ssl_method_st -class ssl_session_st(Structure): - pass -class sess_cert_st(Structure): - pass -ssl_session_st._fields_ = [ - ('ssl_version', c_int), - ('key_arg_length', c_uint), - ('key_arg', c_ubyte * 8), - ('master_key_length', c_int), - ('master_key', c_ubyte * 48), - ('session_id_length', c_uint), - ('session_id', c_ubyte * 32), - ('sid_ctx_length', c_uint), - ('sid_ctx', c_ubyte * 32), - ('not_resumable', c_int), - ('sess_cert', POINTER(sess_cert_st)), - ('peer', POINTER(X509)), - ('verify_result', c_long), - ('references', c_int), - ('timeout', c_long), - ('time', c_long), - ('compress_meth', c_int), - ('cipher', POINTER(SSL_CIPHER)), - ('cipher_id', c_ulong), - ('ciphers', POINTER(STACK)), - ('ex_data', CRYPTO_EX_DATA), - ('prev', POINTER(ssl_session_st)), - ('next', POINTER(ssl_session_st)), -] -assert sizeof(ssl_session_st) == 200, sizeof(ssl_session_st) -assert alignment(ssl_session_st) == 4, alignment(ssl_session_st) -sess_cert_st._fields_ = [ -] -SSL_SESSION = ssl_session_st -GEN_SESSION_CB = CFUNCTYPE(c_int, POINTER(SSL), POINTER(c_ubyte), POINTER(c_uint)) -class ssl_comp_st(Structure): - pass -ssl_comp_st._fields_ = [ - ('id', c_int), - ('name', STRING), - ('method', POINTER(COMP_METHOD)), -] -assert sizeof(ssl_comp_st) == 12, sizeof(ssl_comp_st) -assert alignment(ssl_comp_st) == 4, alignment(ssl_comp_st) -SSL_COMP = ssl_comp_st -class N10ssl_ctx_st4DOLLAR_18E(Structure): - pass -N10ssl_ctx_st4DOLLAR_18E._fields_ = [ - ('sess_connect', c_int), - ('sess_connect_renegotiate', c_int), - ('sess_connect_good', c_int), - ('sess_accept', c_int), - ('sess_accept_renegotiate', c_int), - ('sess_accept_good', c_int), - ('sess_miss', c_int), - ('sess_timeout', c_int), - ('sess_cache_full', c_int), - ('sess_hit', c_int), - ('sess_cb_hit', c_int), -] -assert sizeof(N10ssl_ctx_st4DOLLAR_18E) == 44, sizeof(N10ssl_ctx_st4DOLLAR_18E) -assert alignment(N10ssl_ctx_st4DOLLAR_18E) == 4, alignment(N10ssl_ctx_st4DOLLAR_18E) -class cert_st(Structure): - pass -ssl_ctx_st._fields_ = [ - ('method', POINTER(SSL_METHOD)), - ('cipher_list', POINTER(STACK)), - ('cipher_list_by_id', POINTER(STACK)), - ('cert_store', POINTER(x509_store_st)), - ('sessions', POINTER(lhash_st)), - ('session_cache_size', c_ulong), - ('session_cache_head', POINTER(ssl_session_st)), - ('session_cache_tail', POINTER(ssl_session_st)), - ('session_cache_mode', c_int), - ('session_timeout', c_long), - ('new_session_cb', CFUNCTYPE(c_int, POINTER(ssl_st), POINTER(SSL_SESSION))), - ('remove_session_cb', CFUNCTYPE(None, POINTER(ssl_ctx_st), POINTER(SSL_SESSION))), - ('get_session_cb', CFUNCTYPE(POINTER(SSL_SESSION), POINTER(ssl_st), POINTER(c_ubyte), c_int, POINTER(c_int))), - ('stats', N10ssl_ctx_st4DOLLAR_18E), - ('references', c_int), - ('app_verify_callback', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX), c_void_p)), - ('app_verify_arg', c_void_p), - ('default_passwd_callback', POINTER(pem_password_cb)), - ('default_passwd_callback_userdata', c_void_p), - ('client_cert_cb', CFUNCTYPE(c_int, POINTER(SSL), POINTER(POINTER(X509)), POINTER(POINTER(EVP_PKEY)))), - ('ex_data', CRYPTO_EX_DATA), - ('rsa_md5', POINTER(EVP_MD)), - ('md5', POINTER(EVP_MD)), - ('sha1', POINTER(EVP_MD)), - ('extra_certs', POINTER(STACK)), - ('comp_methods', POINTER(STACK)), - ('info_callback', CFUNCTYPE(None, POINTER(SSL), c_int, c_int)), - ('client_CA', POINTER(STACK)), - ('options', c_ulong), - ('mode', c_ulong), - ('max_cert_list', c_long), - ('cert', POINTER(cert_st)), - ('read_ahead', c_int), - ('msg_callback', CFUNCTYPE(None, c_int, c_int, c_int, c_void_p, c_ulong, POINTER(SSL), c_void_p)), - ('msg_callback_arg', c_void_p), - ('verify_mode', c_int), - ('verify_depth', c_int), - ('sid_ctx_length', c_uint), - ('sid_ctx', c_ubyte * 32), - ('default_verify_callback', CFUNCTYPE(c_int, c_int, POINTER(X509_STORE_CTX))), - ('generate_session_id', GEN_SESSION_CB), - ('purpose', c_int), - ('trust', c_int), - ('quiet_shutdown', c_int), -] -assert sizeof(ssl_ctx_st) == 248, sizeof(ssl_ctx_st) -assert alignment(ssl_ctx_st) == 4, alignment(ssl_ctx_st) -cert_st._fields_ = [ -] -class ssl2_state_st(Structure): - pass -class ssl3_state_st(Structure): - pass -ssl_st._fields_ = [ - ('version', c_int), - ('type', c_int), - ('method', POINTER(SSL_METHOD)), - ('rbio', POINTER(BIO)), - ('wbio', POINTER(BIO)), - ('bbio', POINTER(BIO)), - ('rwstate', c_int), - ('in_handshake', c_int), - ('handshake_func', CFUNCTYPE(c_int)), - ('server', c_int), - ('new_session', c_int), - ('quiet_shutdown', c_int), - ('shutdown', c_int), - ('state', c_int), - ('rstate', c_int), - ('init_buf', POINTER(BUF_MEM)), - ('init_msg', c_void_p), - ('init_num', c_int), - ('init_off', c_int), - ('packet', POINTER(c_ubyte)), - ('packet_length', c_uint), - ('s2', POINTER(ssl2_state_st)), - ('s3', POINTER(ssl3_state_st)), - ('read_ahead', c_int), - ('msg_callback', CFUNCTYPE(None, c_int, c_int, c_int, c_void_p, c_ulong, POINTER(SSL), c_void_p)), - ('msg_callback_arg', c_void_p), - ('hit', c_int), - ('purpose', c_int), - ('trust', c_int), - ('cipher_list', POINTER(STACK)), - ('cipher_list_by_id', POINTER(STACK)), - ('enc_read_ctx', POINTER(EVP_CIPHER_CTX)), - ('read_hash', POINTER(EVP_MD)), - ('expand', POINTER(COMP_CTX)), - ('enc_write_ctx', POINTER(EVP_CIPHER_CTX)), - ('write_hash', POINTER(EVP_MD)), - ('compress', POINTER(COMP_CTX)), - ('cert', POINTER(cert_st)), - ('sid_ctx_length', c_uint), - ('sid_ctx', c_ubyte * 32), - ('session', POINTER(SSL_SESSION)), - ('generate_session_id', GEN_SESSION_CB), - ('verify_mode', c_int), - ('verify_depth', c_int), - ('verify_callback', CFUNCTYPE(c_int, c_int, POINTER(X509_STORE_CTX))), - ('info_callback', CFUNCTYPE(None, POINTER(SSL), c_int, c_int)), - ('error', c_int), - ('error_code', c_int), - ('ctx', POINTER(SSL_CTX)), - ('debug', c_int), - ('verify_result', c_long), - ('ex_data', CRYPTO_EX_DATA), - ('client_CA', POINTER(STACK)), - ('references', c_int), - ('options', c_ulong), - ('mode', c_ulong), - ('max_cert_list', c_long), - ('first_packet', c_int), - ('client_version', c_int), -] -assert sizeof(ssl_st) == 268, sizeof(ssl_st) -assert alignment(ssl_st) == 4, alignment(ssl_st) -class N13ssl2_state_st4DOLLAR_19E(Structure): - pass -N13ssl2_state_st4DOLLAR_19E._fields_ = [ - ('conn_id_length', c_uint), - ('cert_type', c_uint), - ('cert_length', c_uint), - ('csl', c_uint), - ('clear', c_uint), - ('enc', c_uint), - ('ccl', c_ubyte * 32), - ('cipher_spec_length', c_uint), - ('session_id_length', c_uint), - ('clen', c_uint), - ('rlen', c_uint), -] -assert sizeof(N13ssl2_state_st4DOLLAR_19E) == 72, sizeof(N13ssl2_state_st4DOLLAR_19E) -assert alignment(N13ssl2_state_st4DOLLAR_19E) == 4, alignment(N13ssl2_state_st4DOLLAR_19E) -ssl2_state_st._fields_ = [ - ('three_byte_header', c_int), - ('clear_text', c_int), - ('escape', c_int), - ('ssl2_rollback', c_int), - ('wnum', c_uint), - ('wpend_tot', c_int), - ('wpend_buf', POINTER(c_ubyte)), - ('wpend_off', c_int), - ('wpend_len', c_int), - ('wpend_ret', c_int), - ('rbuf_left', c_int), - ('rbuf_offs', c_int), - ('rbuf', POINTER(c_ubyte)), - ('wbuf', POINTER(c_ubyte)), - ('write_ptr', POINTER(c_ubyte)), - ('padding', c_uint), - ('rlength', c_uint), - ('ract_data_length', c_int), - ('wlength', c_uint), - ('wact_data_length', c_int), - ('ract_data', POINTER(c_ubyte)), - ('wact_data', POINTER(c_ubyte)), - ('mac_data', POINTER(c_ubyte)), - ('read_key', POINTER(c_ubyte)), - ('write_key', POINTER(c_ubyte)), - ('challenge_length', c_uint), - ('challenge', c_ubyte * 32), - ('conn_id_length', c_uint), - ('conn_id', c_ubyte * 16), - ('key_material_length', c_uint), - ('key_material', c_ubyte * 48), - ('read_sequence', c_ulong), - ('write_sequence', c_ulong), - ('tmp', N13ssl2_state_st4DOLLAR_19E), -] -assert sizeof(ssl2_state_st) == 288, sizeof(ssl2_state_st) -assert alignment(ssl2_state_st) == 4, alignment(ssl2_state_st) -SSL2_STATE = ssl2_state_st -class ssl3_record_st(Structure): - pass -ssl3_record_st._fields_ = [ - ('type', c_int), - ('length', c_uint), - ('off', c_uint), - ('data', POINTER(c_ubyte)), - ('input', POINTER(c_ubyte)), - ('comp', POINTER(c_ubyte)), -] -assert sizeof(ssl3_record_st) == 24, sizeof(ssl3_record_st) -assert alignment(ssl3_record_st) == 4, alignment(ssl3_record_st) -SSL3_RECORD = ssl3_record_st -class ssl3_buffer_st(Structure): - pass -size_t = __darwin_size_t -ssl3_buffer_st._fields_ = [ - ('buf', POINTER(c_ubyte)), - ('len', size_t), - ('offset', c_int), - ('left', c_int), -] -assert sizeof(ssl3_buffer_st) == 16, sizeof(ssl3_buffer_st) -assert alignment(ssl3_buffer_st) == 4, alignment(ssl3_buffer_st) -SSL3_BUFFER = ssl3_buffer_st -class N13ssl3_state_st4DOLLAR_20E(Structure): - pass -N13ssl3_state_st4DOLLAR_20E._fields_ = [ - ('cert_verify_md', c_ubyte * 72), - ('finish_md', c_ubyte * 72), - ('finish_md_len', c_int), - ('peer_finish_md', c_ubyte * 72), - ('peer_finish_md_len', c_int), - ('message_size', c_ulong), - ('message_type', c_int), - ('new_cipher', POINTER(SSL_CIPHER)), - ('dh', POINTER(DH)), - ('next_state', c_int), - ('reuse_message', c_int), - ('cert_req', c_int), - ('ctype_num', c_int), - ('ctype', c_char * 7), - ('ca_names', POINTER(STACK)), - ('use_rsa_tmp', c_int), - ('key_block_length', c_int), - ('key_block', POINTER(c_ubyte)), - ('new_sym_enc', POINTER(EVP_CIPHER)), - ('new_hash', POINTER(EVP_MD)), - ('new_compression', POINTER(SSL_COMP)), - ('cert_request', c_int), -] -assert sizeof(N13ssl3_state_st4DOLLAR_20E) == 296, sizeof(N13ssl3_state_st4DOLLAR_20E) -assert alignment(N13ssl3_state_st4DOLLAR_20E) == 4, alignment(N13ssl3_state_st4DOLLAR_20E) -ssl3_state_st._fields_ = [ - ('flags', c_long), - ('delay_buf_pop_ret', c_int), - ('read_sequence', c_ubyte * 8), - ('read_mac_secret', c_ubyte * 36), - ('write_sequence', c_ubyte * 8), - ('write_mac_secret', c_ubyte * 36), - ('server_random', c_ubyte * 32), - ('client_random', c_ubyte * 32), - ('need_empty_fragments', c_int), - ('empty_fragment_done', c_int), - ('rbuf', SSL3_BUFFER), - ('wbuf', SSL3_BUFFER), - ('rrec', SSL3_RECORD), - ('wrec', SSL3_RECORD), - ('alert_fragment', c_ubyte * 2), - ('alert_fragment_len', c_uint), - ('handshake_fragment', c_ubyte * 4), - ('handshake_fragment_len', c_uint), - ('wnum', c_uint), - ('wpend_tot', c_int), - ('wpend_type', c_int), - ('wpend_ret', c_int), - ('wpend_buf', POINTER(c_ubyte)), - ('finish_dgst1', EVP_MD_CTX), - ('finish_dgst2', EVP_MD_CTX), - ('change_cipher_spec', c_int), - ('warn_alert', c_int), - ('fatal_alert', c_int), - ('alert_dispatch', c_int), - ('send_alert', c_ubyte * 2), - ('renegotiate', c_int), - ('total_renegotiations', c_int), - ('num_renegotiations', c_int), - ('in_read_app_data', c_int), - ('tmp', N13ssl3_state_st4DOLLAR_20E), -] -assert sizeof(ssl3_state_st) == 648, sizeof(ssl3_state_st) -assert alignment(ssl3_state_st) == 4, alignment(ssl3_state_st) -SSL3_STATE = ssl3_state_st -stack_st._fields_ = [ - ('num', c_int), - ('data', POINTER(STRING)), - ('sorted', c_int), - ('num_alloc', c_int), - ('comp', CFUNCTYPE(c_int, POINTER(STRING), POINTER(STRING))), -] -assert sizeof(stack_st) == 20, sizeof(stack_st) -assert alignment(stack_st) == 4, alignment(stack_st) -class ui_st(Structure): - pass -ui_st._fields_ = [ -] -UI = ui_st -class ui_method_st(Structure): - pass -ui_method_st._fields_ = [ -] -UI_METHOD = ui_method_st -class ui_string_st(Structure): - pass -ui_string_st._fields_ = [ -] -UI_STRING = ui_string_st - -# values for enumeration 'UI_string_types' -UI_string_types = c_int # enum -class X509_objects_st(Structure): - pass -X509_objects_st._fields_ = [ - ('nid', c_int), - ('a2i', CFUNCTYPE(c_int)), - ('i2a', CFUNCTYPE(c_int)), -] -assert sizeof(X509_objects_st) == 12, sizeof(X509_objects_st) -assert alignment(X509_objects_st) == 4, alignment(X509_objects_st) -X509_OBJECTS = X509_objects_st -X509_algor_st._fields_ = [ - ('algorithm', POINTER(ASN1_OBJECT)), - ('parameter', POINTER(ASN1_TYPE)), -] -assert sizeof(X509_algor_st) == 8, sizeof(X509_algor_st) -assert alignment(X509_algor_st) == 4, alignment(X509_algor_st) -class X509_val_st(Structure): - pass -X509_val_st._fields_ = [ - ('notBefore', POINTER(ASN1_TIME)), - ('notAfter', POINTER(ASN1_TIME)), -] -assert sizeof(X509_val_st) == 8, sizeof(X509_val_st) -assert alignment(X509_val_st) == 4, alignment(X509_val_st) -X509_VAL = X509_val_st -class X509_pubkey_st(Structure): - pass -X509_pubkey_st._fields_ = [ - ('algor', POINTER(X509_ALGOR)), - ('public_key', POINTER(ASN1_BIT_STRING)), - ('pkey', POINTER(EVP_PKEY)), -] -assert sizeof(X509_pubkey_st) == 12, sizeof(X509_pubkey_st) -assert alignment(X509_pubkey_st) == 4, alignment(X509_pubkey_st) -X509_PUBKEY = X509_pubkey_st -class X509_sig_st(Structure): - pass -X509_sig_st._fields_ = [ - ('algor', POINTER(X509_ALGOR)), - ('digest', POINTER(ASN1_OCTET_STRING)), -] -assert sizeof(X509_sig_st) == 8, sizeof(X509_sig_st) -assert alignment(X509_sig_st) == 4, alignment(X509_sig_st) -X509_SIG = X509_sig_st -class X509_name_entry_st(Structure): - pass -X509_name_entry_st._fields_ = [ - ('object', POINTER(ASN1_OBJECT)), - ('value', POINTER(ASN1_STRING)), - ('set', c_int), - ('size', c_int), -] -assert sizeof(X509_name_entry_st) == 16, sizeof(X509_name_entry_st) -assert alignment(X509_name_entry_st) == 4, alignment(X509_name_entry_st) -X509_NAME_ENTRY = X509_name_entry_st -X509_name_st._fields_ = [ - ('entries', POINTER(STACK)), - ('modified', c_int), - ('bytes', POINTER(BUF_MEM)), - ('hash', c_ulong), -] -assert sizeof(X509_name_st) == 16, sizeof(X509_name_st) -assert alignment(X509_name_st) == 4, alignment(X509_name_st) -class X509_extension_st(Structure): - pass -X509_extension_st._fields_ = [ - ('object', POINTER(ASN1_OBJECT)), - ('critical', ASN1_BOOLEAN), - ('value', POINTER(ASN1_OCTET_STRING)), -] -assert sizeof(X509_extension_st) == 12, sizeof(X509_extension_st) -assert alignment(X509_extension_st) == 4, alignment(X509_extension_st) -X509_EXTENSION = X509_extension_st -class x509_attributes_st(Structure): - pass -class N18x509_attributes_st4DOLLAR_13E(Union): - pass -N18x509_attributes_st4DOLLAR_13E._fields_ = [ - ('ptr', STRING), - ('set', POINTER(STACK)), - ('single', POINTER(ASN1_TYPE)), -] -assert sizeof(N18x509_attributes_st4DOLLAR_13E) == 4, sizeof(N18x509_attributes_st4DOLLAR_13E) -assert alignment(N18x509_attributes_st4DOLLAR_13E) == 4, alignment(N18x509_attributes_st4DOLLAR_13E) -x509_attributes_st._fields_ = [ - ('object', POINTER(ASN1_OBJECT)), - ('single', c_int), - ('value', N18x509_attributes_st4DOLLAR_13E), -] -assert sizeof(x509_attributes_st) == 12, sizeof(x509_attributes_st) -assert alignment(x509_attributes_st) == 4, alignment(x509_attributes_st) -X509_ATTRIBUTE = x509_attributes_st -class X509_req_info_st(Structure): - pass -X509_req_info_st._fields_ = [ - ('enc', ASN1_ENCODING), - ('version', POINTER(ASN1_INTEGER)), - ('subject', POINTER(X509_NAME)), - ('pubkey', POINTER(X509_PUBKEY)), - ('attributes', POINTER(STACK)), -] -assert sizeof(X509_req_info_st) == 28, sizeof(X509_req_info_st) -assert alignment(X509_req_info_st) == 4, alignment(X509_req_info_st) -X509_REQ_INFO = X509_req_info_st -class X509_req_st(Structure): - pass -X509_req_st._fields_ = [ - ('req_info', POINTER(X509_REQ_INFO)), - ('sig_alg', POINTER(X509_ALGOR)), - ('signature', POINTER(ASN1_BIT_STRING)), - ('references', c_int), -] -assert sizeof(X509_req_st) == 16, sizeof(X509_req_st) -assert alignment(X509_req_st) == 4, alignment(X509_req_st) -X509_REQ = X509_req_st -class x509_cinf_st(Structure): - pass -x509_cinf_st._fields_ = [ - ('version', POINTER(ASN1_INTEGER)), - ('serialNumber', POINTER(ASN1_INTEGER)), - ('signature', POINTER(X509_ALGOR)), - ('issuer', POINTER(X509_NAME)), - ('validity', POINTER(X509_VAL)), - ('subject', POINTER(X509_NAME)), - ('key', POINTER(X509_PUBKEY)), - ('issuerUID', POINTER(ASN1_BIT_STRING)), - ('subjectUID', POINTER(ASN1_BIT_STRING)), - ('extensions', POINTER(STACK)), -] -assert sizeof(x509_cinf_st) == 40, sizeof(x509_cinf_st) -assert alignment(x509_cinf_st) == 4, alignment(x509_cinf_st) -X509_CINF = x509_cinf_st -class x509_cert_aux_st(Structure): - pass -x509_cert_aux_st._fields_ = [ - ('trust', POINTER(STACK)), - ('reject', POINTER(STACK)), - ('alias', POINTER(ASN1_UTF8STRING)), - ('keyid', POINTER(ASN1_OCTET_STRING)), - ('other', POINTER(STACK)), -] -assert sizeof(x509_cert_aux_st) == 20, sizeof(x509_cert_aux_st) -assert alignment(x509_cert_aux_st) == 4, alignment(x509_cert_aux_st) -X509_CERT_AUX = x509_cert_aux_st -class AUTHORITY_KEYID_st(Structure): - pass -x509_st._fields_ = [ - ('cert_info', POINTER(X509_CINF)), - ('sig_alg', POINTER(X509_ALGOR)), - ('signature', POINTER(ASN1_BIT_STRING)), - ('valid', c_int), - ('references', c_int), - ('name', STRING), - ('ex_data', CRYPTO_EX_DATA), - ('ex_pathlen', c_long), - ('ex_flags', c_ulong), - ('ex_kusage', c_ulong), - ('ex_xkusage', c_ulong), - ('ex_nscert', c_ulong), - ('skid', POINTER(ASN1_OCTET_STRING)), - ('akid', POINTER(AUTHORITY_KEYID_st)), - ('sha1_hash', c_ubyte * 20), - ('aux', POINTER(X509_CERT_AUX)), -] -assert sizeof(x509_st) == 84, sizeof(x509_st) -assert alignment(x509_st) == 4, alignment(x509_st) -AUTHORITY_KEYID_st._fields_ = [ -] -class x509_trust_st(Structure): - pass -x509_trust_st._fields_ = [ - ('trust', c_int), - ('flags', c_int), - ('check_trust', CFUNCTYPE(c_int, POINTER(x509_trust_st), POINTER(X509), c_int)), - ('name', STRING), - ('arg1', c_int), - ('arg2', c_void_p), -] -assert sizeof(x509_trust_st) == 24, sizeof(x509_trust_st) -assert alignment(x509_trust_st) == 4, alignment(x509_trust_st) -X509_TRUST = x509_trust_st -class X509_revoked_st(Structure): - pass -X509_revoked_st._fields_ = [ - ('serialNumber', POINTER(ASN1_INTEGER)), - ('revocationDate', POINTER(ASN1_TIME)), - ('extensions', POINTER(STACK)), - ('sequence', c_int), -] -assert sizeof(X509_revoked_st) == 16, sizeof(X509_revoked_st) -assert alignment(X509_revoked_st) == 4, alignment(X509_revoked_st) -X509_REVOKED = X509_revoked_st -class X509_crl_info_st(Structure): - pass -X509_crl_info_st._fields_ = [ - ('version', POINTER(ASN1_INTEGER)), - ('sig_alg', POINTER(X509_ALGOR)), - ('issuer', POINTER(X509_NAME)), - ('lastUpdate', POINTER(ASN1_TIME)), - ('nextUpdate', POINTER(ASN1_TIME)), - ('revoked', POINTER(STACK)), - ('extensions', POINTER(STACK)), - ('enc', ASN1_ENCODING), -] -assert sizeof(X509_crl_info_st) == 40, sizeof(X509_crl_info_st) -assert alignment(X509_crl_info_st) == 4, alignment(X509_crl_info_st) -X509_CRL_INFO = X509_crl_info_st -X509_crl_st._fields_ = [ - ('crl', POINTER(X509_CRL_INFO)), - ('sig_alg', POINTER(X509_ALGOR)), - ('signature', POINTER(ASN1_BIT_STRING)), - ('references', c_int), -] -assert sizeof(X509_crl_st) == 16, sizeof(X509_crl_st) -assert alignment(X509_crl_st) == 4, alignment(X509_crl_st) -class private_key_st(Structure): - pass -private_key_st._fields_ = [ - ('version', c_int), - ('enc_algor', POINTER(X509_ALGOR)), - ('enc_pkey', POINTER(ASN1_OCTET_STRING)), - ('dec_pkey', POINTER(EVP_PKEY)), - ('key_length', c_int), - ('key_data', STRING), - ('key_free', c_int), - ('cipher', EVP_CIPHER_INFO), - ('references', c_int), -] -assert sizeof(private_key_st) == 52, sizeof(private_key_st) -assert alignment(private_key_st) == 4, alignment(private_key_st) -X509_PKEY = private_key_st -class X509_info_st(Structure): - pass -X509_info_st._fields_ = [ - ('x509', POINTER(X509)), - ('crl', POINTER(X509_CRL)), - ('x_pkey', POINTER(X509_PKEY)), - ('enc_cipher', EVP_CIPHER_INFO), - ('enc_len', c_int), - ('enc_data', STRING), - ('references', c_int), -] -assert sizeof(X509_info_st) == 44, sizeof(X509_info_st) -assert alignment(X509_info_st) == 4, alignment(X509_info_st) -X509_INFO = X509_info_st -class Netscape_spkac_st(Structure): - pass -Netscape_spkac_st._fields_ = [ - ('pubkey', POINTER(X509_PUBKEY)), - ('challenge', POINTER(ASN1_IA5STRING)), -] -assert sizeof(Netscape_spkac_st) == 8, sizeof(Netscape_spkac_st) -assert alignment(Netscape_spkac_st) == 4, alignment(Netscape_spkac_st) -NETSCAPE_SPKAC = Netscape_spkac_st -class Netscape_spki_st(Structure): - pass -Netscape_spki_st._fields_ = [ - ('spkac', POINTER(NETSCAPE_SPKAC)), - ('sig_algor', POINTER(X509_ALGOR)), - ('signature', POINTER(ASN1_BIT_STRING)), -] -assert sizeof(Netscape_spki_st) == 12, sizeof(Netscape_spki_st) -assert alignment(Netscape_spki_st) == 4, alignment(Netscape_spki_st) -NETSCAPE_SPKI = Netscape_spki_st -class Netscape_certificate_sequence(Structure): - pass -Netscape_certificate_sequence._fields_ = [ - ('type', POINTER(ASN1_OBJECT)), - ('certs', POINTER(STACK)), -] -assert sizeof(Netscape_certificate_sequence) == 8, sizeof(Netscape_certificate_sequence) -assert alignment(Netscape_certificate_sequence) == 4, alignment(Netscape_certificate_sequence) -NETSCAPE_CERT_SEQUENCE = Netscape_certificate_sequence -class PBEPARAM_st(Structure): - pass -PBEPARAM_st._fields_ = [ - ('salt', POINTER(ASN1_OCTET_STRING)), - ('iter', POINTER(ASN1_INTEGER)), -] -assert sizeof(PBEPARAM_st) == 8, sizeof(PBEPARAM_st) -assert alignment(PBEPARAM_st) == 4, alignment(PBEPARAM_st) -PBEPARAM = PBEPARAM_st -class PBE2PARAM_st(Structure): - pass -PBE2PARAM_st._fields_ = [ - ('keyfunc', POINTER(X509_ALGOR)), - ('encryption', POINTER(X509_ALGOR)), -] -assert sizeof(PBE2PARAM_st) == 8, sizeof(PBE2PARAM_st) -assert alignment(PBE2PARAM_st) == 4, alignment(PBE2PARAM_st) -PBE2PARAM = PBE2PARAM_st -class PBKDF2PARAM_st(Structure): - pass -PBKDF2PARAM_st._fields_ = [ - ('salt', POINTER(ASN1_TYPE)), - ('iter', POINTER(ASN1_INTEGER)), - ('keylength', POINTER(ASN1_INTEGER)), - ('prf', POINTER(X509_ALGOR)), -] -assert sizeof(PBKDF2PARAM_st) == 16, sizeof(PBKDF2PARAM_st) -assert alignment(PBKDF2PARAM_st) == 4, alignment(PBKDF2PARAM_st) -PBKDF2PARAM = PBKDF2PARAM_st -class pkcs8_priv_key_info_st(Structure): - pass -pkcs8_priv_key_info_st._fields_ = [ - ('broken', c_int), - ('version', POINTER(ASN1_INTEGER)), - ('pkeyalg', POINTER(X509_ALGOR)), - ('pkey', POINTER(ASN1_TYPE)), - ('attributes', POINTER(STACK)), -] -assert sizeof(pkcs8_priv_key_info_st) == 20, sizeof(pkcs8_priv_key_info_st) -assert alignment(pkcs8_priv_key_info_st) == 4, alignment(pkcs8_priv_key_info_st) -PKCS8_PRIV_KEY_INFO = pkcs8_priv_key_info_st -class x509_hash_dir_st(Structure): - pass -x509_hash_dir_st._fields_ = [ - ('num_dirs', c_int), - ('dirs', POINTER(STRING)), - ('dirs_type', POINTER(c_int)), - ('num_dirs_alloced', c_int), -] -assert sizeof(x509_hash_dir_st) == 16, sizeof(x509_hash_dir_st) -assert alignment(x509_hash_dir_st) == 4, alignment(x509_hash_dir_st) -X509_HASH_DIR_CTX = x509_hash_dir_st -class x509_file_st(Structure): - pass -x509_file_st._fields_ = [ - ('num_paths', c_int), - ('num_alloced', c_int), - ('paths', POINTER(STRING)), - ('path_type', POINTER(c_int)), -] -assert sizeof(x509_file_st) == 16, sizeof(x509_file_st) -assert alignment(x509_file_st) == 4, alignment(x509_file_st) -X509_CERT_FILE_CTX = x509_file_st -class x509_object_st(Structure): - pass -class N14x509_object_st4DOLLAR_14E(Union): - pass -N14x509_object_st4DOLLAR_14E._fields_ = [ - ('ptr', STRING), - ('x509', POINTER(X509)), - ('crl', POINTER(X509_CRL)), - ('pkey', POINTER(EVP_PKEY)), -] -assert sizeof(N14x509_object_st4DOLLAR_14E) == 4, sizeof(N14x509_object_st4DOLLAR_14E) -assert alignment(N14x509_object_st4DOLLAR_14E) == 4, alignment(N14x509_object_st4DOLLAR_14E) -x509_object_st._fields_ = [ - ('type', c_int), - ('data', N14x509_object_st4DOLLAR_14E), -] -assert sizeof(x509_object_st) == 8, sizeof(x509_object_st) -assert alignment(x509_object_st) == 4, alignment(x509_object_st) -X509_OBJECT = x509_object_st -class x509_lookup_st(Structure): - pass -X509_LOOKUP = x509_lookup_st -class x509_lookup_method_st(Structure): - pass -x509_lookup_method_st._fields_ = [ - ('name', STRING), - ('new_item', CFUNCTYPE(c_int, POINTER(X509_LOOKUP))), - ('free', CFUNCTYPE(None, POINTER(X509_LOOKUP))), - ('init', CFUNCTYPE(c_int, POINTER(X509_LOOKUP))), - ('shutdown', CFUNCTYPE(c_int, POINTER(X509_LOOKUP))), - ('ctrl', CFUNCTYPE(c_int, POINTER(X509_LOOKUP), c_int, STRING, c_long, POINTER(STRING))), - ('get_by_subject', CFUNCTYPE(c_int, POINTER(X509_LOOKUP), c_int, POINTER(X509_NAME), POINTER(X509_OBJECT))), - ('get_by_issuer_serial', CFUNCTYPE(c_int, POINTER(X509_LOOKUP), c_int, POINTER(X509_NAME), POINTER(ASN1_INTEGER), POINTER(X509_OBJECT))), - ('get_by_fingerprint', CFUNCTYPE(c_int, POINTER(X509_LOOKUP), c_int, POINTER(c_ubyte), c_int, POINTER(X509_OBJECT))), - ('get_by_alias', CFUNCTYPE(c_int, POINTER(X509_LOOKUP), c_int, STRING, c_int, POINTER(X509_OBJECT))), -] -assert sizeof(x509_lookup_method_st) == 40, sizeof(x509_lookup_method_st) -assert alignment(x509_lookup_method_st) == 4, alignment(x509_lookup_method_st) -X509_LOOKUP_METHOD = x509_lookup_method_st -x509_store_st._fields_ = [ - ('cache', c_int), - ('objs', POINTER(STACK)), - ('get_cert_methods', POINTER(STACK)), - ('flags', c_ulong), - ('purpose', c_int), - ('trust', c_int), - ('verify', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX))), - ('verify_cb', CFUNCTYPE(c_int, c_int, POINTER(X509_STORE_CTX))), - ('get_issuer', CFUNCTYPE(c_int, POINTER(POINTER(X509)), POINTER(X509_STORE_CTX), POINTER(X509))), - ('check_issued', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX), POINTER(X509), POINTER(X509))), - ('check_revocation', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX))), - ('get_crl', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX), POINTER(POINTER(X509_CRL)), POINTER(X509))), - ('check_crl', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX), POINTER(X509_CRL))), - ('cert_crl', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX), POINTER(X509_CRL), POINTER(X509))), - ('cleanup', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX))), - ('ex_data', CRYPTO_EX_DATA), - ('references', c_int), - ('depth', c_int), -] -assert sizeof(x509_store_st) == 76, sizeof(x509_store_st) -assert alignment(x509_store_st) == 4, alignment(x509_store_st) -x509_lookup_st._fields_ = [ - ('init', c_int), - ('skip', c_int), - ('method', POINTER(X509_LOOKUP_METHOD)), - ('method_data', STRING), - ('store_ctx', POINTER(X509_STORE)), -] -assert sizeof(x509_lookup_st) == 20, sizeof(x509_lookup_st) -assert alignment(x509_lookup_st) == 4, alignment(x509_lookup_st) -time_t = __darwin_time_t -x509_store_ctx_st._fields_ = [ - ('ctx', POINTER(X509_STORE)), - ('current_method', c_int), - ('cert', POINTER(X509)), - ('untrusted', POINTER(STACK)), - ('purpose', c_int), - ('trust', c_int), - ('check_time', time_t), - ('flags', c_ulong), - ('other_ctx', c_void_p), - ('verify', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX))), - ('verify_cb', CFUNCTYPE(c_int, c_int, POINTER(X509_STORE_CTX))), - ('get_issuer', CFUNCTYPE(c_int, POINTER(POINTER(X509)), POINTER(X509_STORE_CTX), POINTER(X509))), - ('check_issued', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX), POINTER(X509), POINTER(X509))), - ('check_revocation', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX))), - ('get_crl', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX), POINTER(POINTER(X509_CRL)), POINTER(X509))), - ('check_crl', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX), POINTER(X509_CRL))), - ('cert_crl', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX), POINTER(X509_CRL), POINTER(X509))), - ('cleanup', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX))), - ('depth', c_int), - ('valid', c_int), - ('last_untrusted', c_int), - ('chain', POINTER(STACK)), - ('error_depth', c_int), - ('error', c_int), - ('current_cert', POINTER(X509)), - ('current_issuer', POINTER(X509)), - ('current_crl', POINTER(X509_CRL)), - ('ex_data', CRYPTO_EX_DATA), -] -assert sizeof(x509_store_ctx_st) == 116, sizeof(x509_store_ctx_st) -assert alignment(x509_store_ctx_st) == 4, alignment(x509_store_ctx_st) -va_list = __darwin_va_list -__darwin_off_t = __int64_t -fpos_t = __darwin_off_t -class __sbuf(Structure): - pass -__sbuf._fields_ = [ - ('_base', POINTER(c_ubyte)), - ('_size', c_int), -] -assert sizeof(__sbuf) == 8, sizeof(__sbuf) -assert alignment(__sbuf) == 4, alignment(__sbuf) -class __sFILEX(Structure): - pass -__sFILEX._fields_ = [ -] -class __sFILE(Structure): - pass -__sFILE._pack_ = 4 -__sFILE._fields_ = [ - ('_p', POINTER(c_ubyte)), - ('_r', c_int), - ('_w', c_int), - ('_flags', c_short), - ('_file', c_short), - ('_bf', __sbuf), - ('_lbfsize', c_int), - ('_cookie', c_void_p), - ('_close', CFUNCTYPE(c_int, c_void_p)), - ('_read', CFUNCTYPE(c_int, c_void_p, STRING, c_int)), - ('_seek', CFUNCTYPE(fpos_t, c_void_p, c_longlong, c_int)), - ('_write', CFUNCTYPE(c_int, c_void_p, STRING, c_int)), - ('_ub', __sbuf), - ('_extra', POINTER(__sFILEX)), - ('_ur', c_int), - ('_ubuf', c_ubyte * 3), - ('_nbuf', c_ubyte * 1), - ('_lb', __sbuf), - ('_blksize', c_int), - ('_offset', fpos_t), -] -assert sizeof(__sFILE) == 88, sizeof(__sFILE) -assert alignment(__sFILE) == 4, alignment(__sFILE) -FILE = __sFILE -ct_rune_t = __darwin_ct_rune_t -rune_t = __darwin_rune_t -class div_t(Structure): - pass -div_t._fields_ = [ - ('quot', c_int), - ('rem', c_int), -] -assert sizeof(div_t) == 8, sizeof(div_t) -assert alignment(div_t) == 4, alignment(div_t) -class ldiv_t(Structure): - pass -ldiv_t._fields_ = [ - ('quot', c_long), - ('rem', c_long), -] -assert sizeof(ldiv_t) == 8, sizeof(ldiv_t) -assert alignment(ldiv_t) == 4, alignment(ldiv_t) -class lldiv_t(Structure): - pass -lldiv_t._pack_ = 4 -lldiv_t._fields_ = [ - ('quot', c_longlong), - ('rem', c_longlong), -] -assert sizeof(lldiv_t) == 16, sizeof(lldiv_t) -assert alignment(lldiv_t) == 4, alignment(lldiv_t) -__darwin_dev_t = __int32_t -dev_t = __darwin_dev_t -__darwin_mode_t = __uint16_t -mode_t = __darwin_mode_t -class mcontext(Structure): - pass -mcontext._fields_ = [ -] -class mcontext64(Structure): - pass -mcontext64._fields_ = [ -] -class __darwin_pthread_handler_rec(Structure): - pass -__darwin_pthread_handler_rec._fields_ = [ - ('__routine', CFUNCTYPE(None, c_void_p)), - ('__arg', c_void_p), - ('__next', POINTER(__darwin_pthread_handler_rec)), -] -assert sizeof(__darwin_pthread_handler_rec) == 12, sizeof(__darwin_pthread_handler_rec) -assert alignment(__darwin_pthread_handler_rec) == 4, alignment(__darwin_pthread_handler_rec) -class _opaque_pthread_attr_t(Structure): - pass -_opaque_pthread_attr_t._fields_ = [ - ('__sig', c_long), - ('__opaque', c_char * 36), -] -assert sizeof(_opaque_pthread_attr_t) == 40, sizeof(_opaque_pthread_attr_t) -assert alignment(_opaque_pthread_attr_t) == 4, alignment(_opaque_pthread_attr_t) -class _opaque_pthread_cond_t(Structure): - pass -_opaque_pthread_cond_t._fields_ = [ - ('__sig', c_long), - ('__opaque', c_char * 24), -] -assert sizeof(_opaque_pthread_cond_t) == 28, sizeof(_opaque_pthread_cond_t) -assert alignment(_opaque_pthread_cond_t) == 4, alignment(_opaque_pthread_cond_t) -class _opaque_pthread_condattr_t(Structure): - pass -_opaque_pthread_condattr_t._fields_ = [ - ('__sig', c_long), - ('__opaque', c_char * 4), -] -assert sizeof(_opaque_pthread_condattr_t) == 8, sizeof(_opaque_pthread_condattr_t) -assert alignment(_opaque_pthread_condattr_t) == 4, alignment(_opaque_pthread_condattr_t) -class _opaque_pthread_mutex_t(Structure): - pass -_opaque_pthread_mutex_t._fields_ = [ - ('__sig', c_long), - ('__opaque', c_char * 40), -] -assert sizeof(_opaque_pthread_mutex_t) == 44, sizeof(_opaque_pthread_mutex_t) -assert alignment(_opaque_pthread_mutex_t) == 4, alignment(_opaque_pthread_mutex_t) -class _opaque_pthread_mutexattr_t(Structure): - pass -_opaque_pthread_mutexattr_t._fields_ = [ - ('__sig', c_long), - ('__opaque', c_char * 8), -] -assert sizeof(_opaque_pthread_mutexattr_t) == 12, sizeof(_opaque_pthread_mutexattr_t) -assert alignment(_opaque_pthread_mutexattr_t) == 4, alignment(_opaque_pthread_mutexattr_t) -class _opaque_pthread_once_t(Structure): - pass -_opaque_pthread_once_t._fields_ = [ - ('__sig', c_long), - ('__opaque', c_char * 4), -] -assert sizeof(_opaque_pthread_once_t) == 8, sizeof(_opaque_pthread_once_t) -assert alignment(_opaque_pthread_once_t) == 4, alignment(_opaque_pthread_once_t) -class _opaque_pthread_rwlock_t(Structure): - pass -_opaque_pthread_rwlock_t._fields_ = [ - ('__sig', c_long), - ('__opaque', c_char * 124), -] -assert sizeof(_opaque_pthread_rwlock_t) == 128, sizeof(_opaque_pthread_rwlock_t) -assert alignment(_opaque_pthread_rwlock_t) == 4, alignment(_opaque_pthread_rwlock_t) -class _opaque_pthread_rwlockattr_t(Structure): - pass -_opaque_pthread_rwlockattr_t._fields_ = [ - ('__sig', c_long), - ('__opaque', c_char * 12), -] -assert sizeof(_opaque_pthread_rwlockattr_t) == 16, sizeof(_opaque_pthread_rwlockattr_t) -assert alignment(_opaque_pthread_rwlockattr_t) == 4, alignment(_opaque_pthread_rwlockattr_t) -class _opaque_pthread_t(Structure): - pass -_opaque_pthread_t._fields_ = [ - ('__sig', c_long), - ('__cleanup_stack', POINTER(__darwin_pthread_handler_rec)), - ('__opaque', c_char * 596), -] -assert sizeof(_opaque_pthread_t) == 604, sizeof(_opaque_pthread_t) -assert alignment(_opaque_pthread_t) == 4, alignment(_opaque_pthread_t) -__darwin_blkcnt_t = __int64_t -__darwin_blksize_t = __int32_t -__darwin_fsblkcnt_t = c_uint -__darwin_fsfilcnt_t = c_uint -__darwin_gid_t = __uint32_t -__darwin_id_t = __uint32_t -__darwin_ino_t = __uint32_t -__darwin_mach_port_name_t = __darwin_natural_t -__darwin_mach_port_t = __darwin_mach_port_name_t -__darwin_mcontext_t = POINTER(mcontext) -__darwin_mcontext64_t = POINTER(mcontext64) -__darwin_pid_t = __int32_t -__darwin_pthread_attr_t = _opaque_pthread_attr_t -__darwin_pthread_cond_t = _opaque_pthread_cond_t -__darwin_pthread_condattr_t = _opaque_pthread_condattr_t -__darwin_pthread_key_t = c_ulong -__darwin_pthread_mutex_t = _opaque_pthread_mutex_t -__darwin_pthread_mutexattr_t = _opaque_pthread_mutexattr_t -__darwin_pthread_once_t = _opaque_pthread_once_t -__darwin_pthread_rwlock_t = _opaque_pthread_rwlock_t -__darwin_pthread_rwlockattr_t = _opaque_pthread_rwlockattr_t -__darwin_pthread_t = POINTER(_opaque_pthread_t) -__darwin_sigset_t = __uint32_t -__darwin_suseconds_t = __int32_t -__darwin_uid_t = __uint32_t -__darwin_useconds_t = __uint32_t -__darwin_uuid_t = c_ubyte * 16 -class sigaltstack(Structure): - pass -sigaltstack._fields_ = [ - ('ss_sp', c_void_p), - ('ss_size', __darwin_size_t), - ('ss_flags', c_int), -] -assert sizeof(sigaltstack) == 12, sizeof(sigaltstack) -assert alignment(sigaltstack) == 4, alignment(sigaltstack) -__darwin_stack_t = sigaltstack -class ucontext(Structure): - pass -ucontext._fields_ = [ - ('uc_onstack', c_int), - ('uc_sigmask', __darwin_sigset_t), - ('uc_stack', __darwin_stack_t), - ('uc_link', POINTER(ucontext)), - ('uc_mcsize', __darwin_size_t), - ('uc_mcontext', __darwin_mcontext_t), -] -assert sizeof(ucontext) == 32, sizeof(ucontext) -assert alignment(ucontext) == 4, alignment(ucontext) -__darwin_ucontext_t = ucontext -class ucontext64(Structure): - pass -ucontext64._fields_ = [ - ('uc_onstack', c_int), - ('uc_sigmask', __darwin_sigset_t), - ('uc_stack', __darwin_stack_t), - ('uc_link', POINTER(ucontext64)), - ('uc_mcsize', __darwin_size_t), - ('uc_mcontext64', __darwin_mcontext64_t), -] -assert sizeof(ucontext64) == 32, sizeof(ucontext64) -assert alignment(ucontext64) == 4, alignment(ucontext64) -__darwin_ucontext64_t = ucontext64 -class timeval(Structure): - pass -timeval._fields_ = [ - ('tv_sec', __darwin_time_t), - ('tv_usec', __darwin_suseconds_t), -] -assert sizeof(timeval) == 8, sizeof(timeval) -assert alignment(timeval) == 4, alignment(timeval) -rlim_t = __int64_t -class rusage(Structure): - pass -rusage._fields_ = [ - ('ru_utime', timeval), - ('ru_stime', timeval), - ('ru_maxrss', c_long), - ('ru_ixrss', c_long), - ('ru_idrss', c_long), - ('ru_isrss', c_long), - ('ru_minflt', c_long), - ('ru_majflt', c_long), - ('ru_nswap', c_long), - ('ru_inblock', c_long), - ('ru_oublock', c_long), - ('ru_msgsnd', c_long), - ('ru_msgrcv', c_long), - ('ru_nsignals', c_long), - ('ru_nvcsw', c_long), - ('ru_nivcsw', c_long), -] -assert sizeof(rusage) == 72, sizeof(rusage) -assert alignment(rusage) == 4, alignment(rusage) -class rlimit(Structure): - pass -rlimit._pack_ = 4 -rlimit._fields_ = [ - ('rlim_cur', rlim_t), - ('rlim_max', rlim_t), -] -assert sizeof(rlimit) == 16, sizeof(rlimit) -assert alignment(rlimit) == 4, alignment(rlimit) -mcontext_t = __darwin_mcontext_t -mcontext64_t = __darwin_mcontext64_t -pthread_attr_t = __darwin_pthread_attr_t -sigset_t = __darwin_sigset_t -ucontext_t = __darwin_ucontext_t -ucontext64_t = __darwin_ucontext64_t -uid_t = __darwin_uid_t -class sigval(Union): - pass -sigval._fields_ = [ - ('sival_int', c_int), - ('sival_ptr', c_void_p), -] -assert sizeof(sigval) == 4, sizeof(sigval) -assert alignment(sigval) == 4, alignment(sigval) -class sigevent(Structure): - pass -sigevent._fields_ = [ - ('sigev_notify', c_int), - ('sigev_signo', c_int), - ('sigev_value', sigval), - ('sigev_notify_function', CFUNCTYPE(None, sigval)), - ('sigev_notify_attributes', POINTER(pthread_attr_t)), -] -assert sizeof(sigevent) == 20, sizeof(sigevent) -assert alignment(sigevent) == 4, alignment(sigevent) -class __siginfo(Structure): - pass -pid_t = __darwin_pid_t -__siginfo._fields_ = [ - ('si_signo', c_int), - ('si_errno', c_int), - ('si_code', c_int), - ('si_pid', pid_t), - ('si_uid', uid_t), - ('si_status', c_int), - ('si_addr', c_void_p), - ('si_value', sigval), - ('si_band', c_long), - ('pad', c_ulong * 7), -] -assert sizeof(__siginfo) == 64, sizeof(__siginfo) -assert alignment(__siginfo) == 4, alignment(__siginfo) -siginfo_t = __siginfo -class __sigaction_u(Union): - pass -__sigaction_u._fields_ = [ - ('__sa_handler', CFUNCTYPE(None, c_int)), - ('__sa_sigaction', CFUNCTYPE(None, c_int, POINTER(__siginfo), c_void_p)), -] -assert sizeof(__sigaction_u) == 4, sizeof(__sigaction_u) -assert alignment(__sigaction_u) == 4, alignment(__sigaction_u) -class __sigaction(Structure): - pass -__sigaction._fields_ = [ - ('__sigaction_u', __sigaction_u), - ('sa_tramp', CFUNCTYPE(None, c_void_p, c_int, c_int, POINTER(siginfo_t), c_void_p)), - ('sa_mask', sigset_t), - ('sa_flags', c_int), -] -assert sizeof(__sigaction) == 16, sizeof(__sigaction) -assert alignment(__sigaction) == 4, alignment(__sigaction) -class sigaction(Structure): - pass -sigaction._fields_ = [ - ('__sigaction_u', __sigaction_u), - ('sa_mask', sigset_t), - ('sa_flags', c_int), -] -assert sizeof(sigaction) == 12, sizeof(sigaction) -assert alignment(sigaction) == 4, alignment(sigaction) -sig_t = CFUNCTYPE(None, c_int) -stack_t = __darwin_stack_t -class sigvec(Structure): - pass -sigvec._fields_ = [ - ('sv_handler', CFUNCTYPE(None, c_int)), - ('sv_mask', c_int), - ('sv_flags', c_int), -] -assert sizeof(sigvec) == 12, sizeof(sigvec) -assert alignment(sigvec) == 4, alignment(sigvec) -class sigstack(Structure): - pass -sigstack._fields_ = [ - ('ss_sp', STRING), - ('ss_onstack', c_int), -] -assert sizeof(sigstack) == 8, sizeof(sigstack) -assert alignment(sigstack) == 4, alignment(sigstack) -u_char = c_ubyte -u_short = c_ushort -u_int = c_uint -u_long = c_ulong -ushort = c_ushort -uint = c_uint -u_quad_t = u_int64_t -quad_t = int64_t -qaddr_t = POINTER(quad_t) -caddr_t = STRING -daddr_t = int32_t -fixpt_t = u_int32_t -blkcnt_t = __darwin_blkcnt_t -blksize_t = __darwin_blksize_t -gid_t = __darwin_gid_t -in_addr_t = __uint32_t -in_port_t = __uint16_t -ino_t = __darwin_ino_t -key_t = __int32_t -nlink_t = __uint16_t -off_t = __darwin_off_t -segsz_t = int32_t -swblk_t = int32_t -clock_t = __darwin_clock_t -ssize_t = __darwin_ssize_t -useconds_t = __darwin_useconds_t -suseconds_t = __darwin_suseconds_t -fd_mask = __int32_t -class fd_set(Structure): - pass -fd_set._fields_ = [ - ('fds_bits', __int32_t * 32), -] -assert sizeof(fd_set) == 128, sizeof(fd_set) -assert alignment(fd_set) == 4, alignment(fd_set) -pthread_cond_t = __darwin_pthread_cond_t -pthread_condattr_t = __darwin_pthread_condattr_t -pthread_mutex_t = __darwin_pthread_mutex_t -pthread_mutexattr_t = __darwin_pthread_mutexattr_t -pthread_once_t = __darwin_pthread_once_t -pthread_rwlock_t = __darwin_pthread_rwlock_t -pthread_rwlockattr_t = __darwin_pthread_rwlockattr_t -pthread_t = __darwin_pthread_t -pthread_key_t = __darwin_pthread_key_t -fsblkcnt_t = __darwin_fsblkcnt_t -fsfilcnt_t = __darwin_fsfilcnt_t - -# values for enumeration 'idtype_t' -idtype_t = c_int # enum -id_t = __darwin_id_t -class wait(Union): - pass -class N4wait3DOLLAR_3E(Structure): - pass -N4wait3DOLLAR_3E._fields_ = [ - ('w_Termsig', c_uint, 7), - ('w_Coredump', c_uint, 1), - ('w_Retcode', c_uint, 8), - ('w_Filler', c_uint, 16), -] -assert sizeof(N4wait3DOLLAR_3E) == 4, sizeof(N4wait3DOLLAR_3E) -assert alignment(N4wait3DOLLAR_3E) == 4, alignment(N4wait3DOLLAR_3E) -class N4wait3DOLLAR_4E(Structure): - pass -N4wait3DOLLAR_4E._fields_ = [ - ('w_Stopval', c_uint, 8), - ('w_Stopsig', c_uint, 8), - ('w_Filler', c_uint, 16), -] -assert sizeof(N4wait3DOLLAR_4E) == 4, sizeof(N4wait3DOLLAR_4E) -assert alignment(N4wait3DOLLAR_4E) == 4, alignment(N4wait3DOLLAR_4E) -wait._fields_ = [ - ('w_status', c_int), - ('w_T', N4wait3DOLLAR_3E), - ('w_S', N4wait3DOLLAR_4E), -] -assert sizeof(wait) == 4, sizeof(wait) -assert alignment(wait) == 4, alignment(wait) -class timespec(Structure): - pass -timespec._fields_ = [ - ('tv_sec', time_t), - ('tv_nsec', c_long), -] -assert sizeof(timespec) == 8, sizeof(timespec) -assert alignment(timespec) == 4, alignment(timespec) -class tm(Structure): - pass -tm._fields_ = [ - ('tm_sec', c_int), - ('tm_min', c_int), - ('tm_hour', c_int), - ('tm_mday', c_int), - ('tm_mon', c_int), - ('tm_year', c_int), - ('tm_wday', c_int), - ('tm_yday', c_int), - ('tm_isdst', c_int), - ('tm_gmtoff', c_long), - ('tm_zone', STRING), -] -assert sizeof(tm) == 44, sizeof(tm) -assert alignment(tm) == 4, alignment(tm) -__gnuc_va_list = STRING -ptrdiff_t = c_int -int8_t = c_byte -int16_t = c_short -uint8_t = c_ubyte -uint16_t = c_ushort -uint32_t = c_uint -uint64_t = c_ulonglong -int_least8_t = int8_t -int_least16_t = int16_t -int_least32_t = int32_t -int_least64_t = int64_t -uint_least8_t = uint8_t -uint_least16_t = uint16_t -uint_least32_t = uint32_t -uint_least64_t = uint64_t -int_fast8_t = int8_t -int_fast16_t = int16_t -int_fast32_t = int32_t -int_fast64_t = int64_t -uint_fast8_t = uint8_t -uint_fast16_t = uint16_t -uint_fast32_t = uint32_t -uint_fast64_t = uint64_t -intptr_t = c_long -uintptr_t = c_ulong -intmax_t = c_longlong -uintmax_t = c_ulonglong -__all__ = ['ENGINE', 'pkcs7_enc_content_st', '__int16_t', - 'X509_REVOKED', 'SSL_CTX', 'UIT_BOOLEAN', - '__darwin_time_t', 'ucontext64_t', 'int_fast32_t', - 'pem_ctx_st', 'uint8_t', 'fpos_t', 'X509', 'COMP_CTX', - 'tm', 'N10pem_ctx_st4DOLLAR_17E', 'swblk_t', - 'ASN1_TEMPLATE', '__darwin_pthread_t', 'fixpt_t', - 'BIO_METHOD', 'ASN1_PRINTABLESTRING', 'EVP_ENCODE_CTX', - 'dh_method', 'bio_f_buffer_ctx_struct', 'in_port_t', - 'X509_SIG', '__darwin_ssize_t', '__darwin_sigset_t', - 'wait', 'uint_fast16_t', 'N12asn1_type_st4DOLLAR_11E', - 'uint_least8_t', 'pthread_rwlock_t', 'ASN1_IA5STRING', - 'fsfilcnt_t', 'ucontext', '__uint64_t', 'timespec', - 'x509_cinf_st', 'COMP_METHOD', 'MD5_CTX', 'buf_mem_st', - 'ASN1_ENCODING_st', 'PBEPARAM', 'X509_NAME_ENTRY', - '__darwin_va_list', 'ucontext_t', 'lhash_st', - 'N4wait3DOLLAR_4E', '__darwin_uuid_t', - '_ossl_old_des_ks_struct', 'id_t', 'ASN1_BIT_STRING', - 'va_list', '__darwin_wchar_t', 'pthread_key_t', - 'pkcs7_signer_info_st', 'ASN1_METHOD', 'DSA_SIG', 'DSA', - 'UIT_NONE', 'pthread_t', '__darwin_useconds_t', - 'uint_fast8_t', 'UI_STRING', 'DES_cblock', - '__darwin_mcontext64_t', 'rlim_t', 'PEM_Encode_Seal_st', - 'SHAstate_st', 'u_quad_t', 'openssl_fptr', - '_opaque_pthread_rwlockattr_t', - 'N18x509_attributes_st4DOLLAR_13E', - '__darwin_pthread_rwlock_t', 'daddr_t', 'ui_string_st', - 'x509_file_st', 'X509_req_info_st', 'int_least64_t', - 'evp_Encode_Ctx_st', 'X509_OBJECTS', 'CRYPTO_EX_DATA', - '__int8_t', 'AUTHORITY_KEYID_st', '_opaque_pthread_attr_t', - 'sigstack', 'EVP_CIPHER_CTX', 'X509_extension_st', 'pid_t', - 'RSA_METHOD', 'PEM_USER', 'pem_recip_st', 'env_md_ctx_st', - 'rc5_key_st', 'ui_st', 'X509_PUBKEY', 'u_int8_t', - 'ASN1_ITEM_st', 'pkcs7_recip_info_st', 'ssl2_state_st', - 'off_t', 'N10ssl_ctx_st4DOLLAR_18E', 'crypto_ex_data_st', - 'ui_method_st', '__darwin_pthread_rwlockattr_t', - 'CRYPTO_EX_dup', '__darwin_ino_t', '__sFILE', - 'OSUnknownByteOrder', 'BN_MONT_CTX', 'ASN1_NULL', 'time_t', - 'CRYPTO_EX_new', 'asn1_type_st', 'CRYPTO_EX_DATA_FUNCS', - 'user_time_t', 'BIGNUM', 'pthread_rwlockattr_t', - 'ASN1_VALUE_st', 'DH_METHOD', '__darwin_off_t', - '_opaque_pthread_t', 'bn_blinding_st', 'RSA', 'ssize_t', - 'mcontext64_t', 'user_long_t', 'fsblkcnt_t', 'cert_st', - '__darwin_pthread_condattr_t', 'X509_PKEY', - '__darwin_id_t', '__darwin_nl_item', 'SSL2_STATE', 'FILE', - 'pthread_mutexattr_t', 'size_t', - '_ossl_old_des_key_schedule', 'pkcs7_issuer_and_serial_st', - 'sigval', 'CRYPTO_MEM_LEAK_CB', 'X509_NAME', 'blkcnt_t', - 'uint_least16_t', '__darwin_dev_t', 'evp_cipher_info_st', - 'BN_BLINDING', 'ssl3_state_st', 'uint_least64_t', - 'user_addr_t', 'DES_key_schedule', 'RIPEMD160_CTX', - 'u_char', 'X509_algor_st', 'uid_t', 'sess_cert_st', - 'u_int64_t', 'u_int16_t', 'sigset_t', '__darwin_ptrdiff_t', - 'ASN1_CTX', 'STACK', '__int32_t', 'UI_METHOD', - 'NETSCAPE_SPKI', 'UIT_PROMPT', 'st_CRYPTO_EX_DATA_IMPL', - 'cast_key_st', 'X509_HASH_DIR_CTX', 'sigevent', - 'user_ssize_t', 'clock_t', 'aes_key_st', - '__darwin_socklen_t', '__darwin_intptr_t', 'int_fast64_t', - 'asn1_string_table_st', 'uint_fast32_t', - 'ASN1_VISIBLESTRING', 'DSA_SIG_st', 'obj_name_st', - 'X509_LOOKUP_METHOD', 'u_int32_t', 'EVP_CIPHER_INFO', - '__gnuc_va_list', 'AES_KEY', 'PKCS7_ISSUER_AND_SERIAL', - 'BN_CTX', '__darwin_blkcnt_t', 'key_t', 'SHA_CTX', - 'pkcs7_signed_st', 'SSL', 'N10pem_ctx_st4DOLLAR_16E', - 'pthread_attr_t', 'EVP_MD', 'uint', 'ASN1_BOOLEAN', - 'ino_t', '__darwin_clock_t', 'ASN1_OCTET_STRING', - 'asn1_ctx_st', 'BIO_F_BUFFER_CTX', 'bn_mont_ctx_st', - 'X509_REQ_INFO', 'PEM_CTX', 'sigvec', - '__darwin_pthread_mutexattr_t', 'x509_attributes_st', - 'stack_t', '__darwin_mode_t', '__mbstate_t', - 'asn1_object_st', 'ASN1_ENCODING', '__uint8_t', - 'LHASH_NODE', 'PKCS7_SIGNER_INFO', 'asn1_method_st', - 'stack_st', 'bio_info_cb', 'div_t', 'UIT_VERIFY', - 'PBEPARAM_st', 'N4wait3DOLLAR_3E', 'quad_t', '__siginfo', - '__darwin_mbstate_t', 'rsa_st', 'ASN1_UNIVERSALSTRING', - 'uint64_t', 'ssl_comp_st', 'X509_OBJECT', 'pthread_cond_t', - 'DH', '__darwin_wctype_t', 'PKCS7_ENVELOPE', 'ASN1_TLC_st', - 'sig_atomic_t', 'BIO', 'nlink_t', 'BUF_MEM', 'SSL3_RECORD', - 'bio_method_st', 'timeval', 'UI_string_types', 'BIO_dummy', - 'ssl_ctx_st', 'NETSCAPE_CERT_SEQUENCE', - 'BIT_STRING_BITNAME_st', '__darwin_pthread_attr_t', - 'int8_t', '__darwin_wint_t', 'OBJ_NAME', - 'PKCS8_PRIV_KEY_INFO', 'PBE2PARAM_st', - 'LHASH_DOALL_FN_TYPE', 'x509_st', 'X509_VAL', 'dev_t', - 'ASN1_TEMPLATE_st', 'MD5state_st', '__uint16_t', - 'LHASH_DOALL_ARG_FN_TYPE', 'mdc2_ctx_st', 'SSL3_STATE', - 'ssl3_buffer_st', 'ASN1_ITEM_EXP', - '_opaque_pthread_condattr_t', 'mode_t', 'ASN1_VALUE', - 'qaddr_t', '__darwin_gid_t', 'EVP_PKEY', 'CRYPTO_EX_free', - '_ossl_old_des_cblock', 'X509_INFO', 'asn1_string_st', - 'intptr_t', 'UIT_INFO', 'int_fast8_t', 'sigaltstack', - 'env_md_st', 'LHASH', '__darwin_ucontext_t', - 'PKCS7_SIGN_ENVELOPE', '__darwin_mcontext_t', 'ct_rune_t', - 'MD2_CTX', 'pthread_once_t', 'SSL3_BUFFER', 'fd_mask', - 'ASN1_TYPE', 'PKCS7_SIGNED', 'ssl3_record_st', 'BF_KEY', - 'MD4state_st', 'MD4_CTX', 'int16_t', 'SSL_CIPHER', - 'rune_t', 'X509_TRUST', 'siginfo_t', 'X509_STORE', - '__sbuf', 'X509_STORE_CTX', '__darwin_blksize_t', 'ldiv_t', - 'ASN1_TIME', 'SSL_METHOD', 'X509_LOOKUP', - 'Netscape_spki_st', 'P_PID', 'sigaction', 'sig_t', - 'hostent', 'x509_cert_aux_st', '_opaque_pthread_cond_t', - 'segsz_t', 'ushort', '__darwin_ct_rune_t', 'fd_set', - 'BN_RECP_CTX', 'x509_lookup_st', 'uint16_t', 'pkcs7_st', - 'asn1_header_st', '__darwin_pthread_key_t', - 'x509_trust_st', '__darwin_pthread_handler_rec', 'int32_t', - 'X509_CRL_INFO', 'N11evp_pkey_st4DOLLAR_12E', 'MDC2_CTX', - 'N23_ossl_old_des_ks_struct4DOLLAR_10E', 'ASN1_HEADER', - 'X509_crl_info_st', 'LHASH_HASH_FN_TYPE', - '_opaque_pthread_mutexattr_t', 'ssl_st', - 'N8pkcs7_st4DOLLAR_15E', 'evp_pkey_st', - 'pkcs7_signedandenveloped_st', '__darwin_mach_port_t', - 'EVP_PBE_KEYGEN', '_opaque_pthread_mutex_t', - 'ASN1_UTCTIME', 'mcontext', 'crypto_ex_data_func_st', - 'u_long', 'PBKDF2PARAM_st', 'rc4_key_st', 'DSA_METHOD', - 'EVP_CIPHER', 'BIT_STRING_BITNAME', 'PKCS7_RECIP_INFO', - 'ssl3_enc_method', 'X509_CERT_AUX', 'uintmax_t', - 'int_fast16_t', 'RC5_32_KEY', 'ucontext64', 'ASN1_INTEGER', - 'u_short', 'N14x509_object_st4DOLLAR_14E', 'mcontext64', - 'X509_sig_st', 'ASN1_GENERALSTRING', 'PKCS7', '__sFILEX', - 'X509_name_entry_st', 'ssl_session_st', 'caddr_t', - 'bignum_st', 'X509_CINF', '__darwin_pthread_cond_t', - 'ASN1_TLC', 'PKCS7_ENCRYPT', 'NETSCAPE_SPKAC', - 'Netscape_spkac_st', 'idtype_t', 'UIT_ERROR', - 'uint_fast64_t', 'in_addr_t', 'pthread_mutex_t', - '__int64_t', 'ASN1_BMPSTRING', 'uint32_t', - 'PEM_ENCODE_SEAL_CTX', 'suseconds_t', 'ASN1_OBJECT', - 'X509_val_st', 'private_key_st', 'CRYPTO_dynlock', - 'X509_objects_st', 'CRYPTO_EX_DATA_IMPL', - 'pthread_condattr_t', 'PKCS7_DIGEST', 'uint_least32_t', - 'ASN1_STRING', '__uint32_t', 'P_PGID', 'rsa_meth_st', - 'X509_crl_st', 'RC2_KEY', '__darwin_fsfilcnt_t', - 'X509_revoked_st', 'PBE2PARAM', 'blksize_t', - 'Netscape_certificate_sequence', 'ssl_cipher_st', - 'bignum_ctx', 'register_t', 'ASN1_UTF8STRING', - 'pkcs7_encrypted_st', 'RC4_KEY', '__darwin_ucontext64_t', - 'N13ssl2_state_st4DOLLAR_19E', 'bn_recp_ctx_st', - 'CAST_KEY', 'X509_ATTRIBUTE', '__darwin_suseconds_t', - '__sigaction', 'user_ulong_t', 'syscall_arg_t', - 'evp_cipher_ctx_st', 'X509_ALGOR', 'mcontext_t', - 'const_DES_cblock', '__darwin_fsblkcnt_t', 'dsa_st', - 'int_least8_t', 'MD2state_st', 'X509_EXTENSION', - 'GEN_SESSION_CB', 'int_least16_t', '__darwin_wctrans_t', - 'PBKDF2PARAM', 'x509_lookup_method_st', 'pem_password_cb', - 'X509_info_st', 'x509_store_st', '__darwin_natural_t', - 'X509_pubkey_st', 'pkcs7_digest_st', '__darwin_size_t', - 'ASN1_STRING_TABLE', 'OSLittleEndian', 'RIPEMD160state_st', - 'pkcs7_enveloped_st', 'UI', 'ptrdiff_t', 'X509_REQ', - 'CRYPTO_dynlock_value', 'X509_req_st', 'x509_store_ctx_st', - 'N13ssl3_state_st4DOLLAR_20E', 'lhash_node_st', - '__darwin_pthread_mutex_t', 'LHASH_COMP_FN_TYPE', - '__darwin_rune_t', 'rlimit', '__darwin_pthread_once_t', - 'OSBigEndian', 'uintptr_t', '__darwin_uid_t', 'u_int', - 'ASN1_T61STRING', 'gid_t', 'ssl_method_st', 'ASN1_ITEM', - 'ASN1_ENUMERATED', '_opaque_pthread_rwlock_t', - 'pkcs8_priv_key_info_st', 'intmax_t', 'sigcontext', - 'X509_CRL', 'rc2_key_st', 'engine_st', 'x509_object_st', - '_opaque_pthread_once_t', 'DES_ks', 'SSL_COMP', - 'dsa_method', 'int64_t', 'bio_st', 'bf_key_st', - 'ASN1_GENERALIZEDTIME', 'PKCS7_ENC_CONTENT', - '__darwin_pid_t', 'lldiv_t', 'comp_method_st', - 'EVP_MD_CTX', 'evp_cipher_st', 'X509_name_st', - 'x509_hash_dir_st', '__darwin_mach_port_name_t', - 'useconds_t', 'user_size_t', 'SSL_SESSION', 'rusage', - 'ssl_crock_st', 'int_least32_t', '__sigaction_u', 'dh_st', - 'P_ALL', '__darwin_stack_t', 'N6DES_ks3DOLLAR_9E', - 'comp_ctx_st', 'X509_CERT_FILE_CTX'] diff --git a/Lib/test/test_lib2to3/data/py2_test_grammar.py b/Lib/test/test_lib2to3/data/py2_test_grammar.py deleted file mode 100644 index 1a631510f4dba7..00000000000000 --- a/Lib/test/test_lib2to3/data/py2_test_grammar.py +++ /dev/null @@ -1,971 +0,0 @@ -# Python test set -- part 1, grammar. -# This just tests whether the parser accepts them all. - -# NOTE: When you run this test as a script from the command line, you -# get warnings about certain hex/oct constants. Since those are -# issued by the parser, you can't suppress them by adding a -# filterwarnings() call to this module. Therefore, to shut up the -# regression test, the filterwarnings() call has been added to -# regrtest.py. - -from test.test_support import check_syntax_error -import unittest -import sys -# testing import * -from sys import * - -class TokenTests(unittest.TestCase): - - def testBackslash(self): - # Backslash means line continuation: - x = 1 \ - + 1 - self.assertEquals(x, 2, 'backslash for line continuation') - - # Backslash does not means continuation in comments :\ - x = 0 - self.assertEquals(x, 0, 'backslash ending comment') - - def testPlainIntegers(self): - self.assertEquals(0xff, 255) - self.assertEquals(0377, 255) - self.assertEquals(2147483647, 017777777777) - # "0x" is not a valid literal - self.assertRaises(SyntaxError, eval, "0x") - from sys import maxint - if maxint == 2147483647: - self.assertEquals(-2147483647-1, -020000000000) - # XXX -2147483648 - self.assert_(037777777777 > 0) - self.assert_(0xffffffff > 0) - for s in '2147483648', '040000000000', '0x100000000': - try: - x = eval(s) - except OverflowError: - self.fail("OverflowError on huge integer literal %r" % s) - elif maxint == 9223372036854775807: - self.assertEquals(-9223372036854775807-1, -01000000000000000000000) - self.assert_(01777777777777777777777 > 0) - self.assert_(0xffffffffffffffff > 0) - for s in '9223372036854775808', '02000000000000000000000', \ - '0x10000000000000000': - try: - x = eval(s) - except OverflowError: - self.fail("OverflowError on huge integer literal %r" % s) - else: - self.fail('Weird maxint value %r' % maxint) - - def testLongIntegers(self): - x = 0L - x = 0l - x = 0xffffffffffffffffL - x = 0xffffffffffffffffl - x = 077777777777777777L - x = 077777777777777777l - x = 123456789012345678901234567890L - x = 123456789012345678901234567890l - - def testFloats(self): - x = 3.14 - x = 314. - x = 0.314 - # XXX x = 000.314 - x = .314 - x = 3e14 - x = 3E14 - x = 3e-14 - x = 3e+14 - x = 3.e14 - x = .3e14 - x = 3.1e4 - - def testStringLiterals(self): - x = ''; y = ""; self.assert_(len(x) == 0 and x == y) - x = '\''; y = "'"; self.assert_(len(x) == 1 and x == y and ord(x) == 39) - x = '"'; y = "\""; self.assert_(len(x) == 1 and x == y and ord(x) == 34) - x = "doesn't \"shrink\" does it" - y = 'doesn\'t "shrink" does it' - self.assert_(len(x) == 24 and x == y) - x = "does \"shrink\" doesn't it" - y = 'does "shrink" doesn\'t it' - self.assert_(len(x) == 24 and x == y) - x = """ -The "quick" -brown fox -jumps over -the 'lazy' dog. -""" - y = '\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n' - self.assertEquals(x, y) - y = ''' -The "quick" -brown fox -jumps over -the 'lazy' dog. -''' - self.assertEquals(x, y) - y = "\n\ -The \"quick\"\n\ -brown fox\n\ -jumps over\n\ -the 'lazy' dog.\n\ -" - self.assertEquals(x, y) - y = '\n\ -The \"quick\"\n\ -brown fox\n\ -jumps over\n\ -the \'lazy\' dog.\n\ -' - self.assertEquals(x, y) - - -class GrammarTests(unittest.TestCase): - - # single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE - # XXX can't test in a script -- this rule is only used when interactive - - # file_input: (NEWLINE | stmt)* ENDMARKER - # Being tested as this very moment this very module - - # expr_input: testlist NEWLINE - # XXX Hard to test -- used only in calls to input() - - def testEvalInput(self): - # testlist ENDMARKER - x = eval('1, 0 or 1') - - def testFuncdef(self): - ### 'def' NAME parameters ':' suite - ### parameters: '(' [varargslist] ')' - ### varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' ('**'|'*' '*') NAME] - ### | ('**'|'*' '*') NAME) - ### | fpdef ['=' test] (',' fpdef ['=' test])* [','] - ### fpdef: NAME | '(' fplist ')' - ### fplist: fpdef (',' fpdef)* [','] - ### arglist: (argument ',')* (argument | *' test [',' '**' test] | '**' test) - ### argument: [test '='] test # Really [keyword '='] test - def f1(): pass - f1() - f1(*()) - f1(*(), **{}) - def f2(one_argument): pass - def f3(two, arguments): pass - def f4(two, (compound, (argument, list))): pass - def f5((compound, first), two): pass - self.assertEquals(f2.func_code.co_varnames, ('one_argument',)) - self.assertEquals(f3.func_code.co_varnames, ('two', 'arguments')) - if sys.platform.startswith('java'): - self.assertEquals(f4.func_code.co_varnames, - ('two', '(compound, (argument, list))', 'compound', 'argument', - 'list',)) - self.assertEquals(f5.func_code.co_varnames, - ('(compound, first)', 'two', 'compound', 'first')) - else: - self.assertEquals(f4.func_code.co_varnames, - ('two', '.1', 'compound', 'argument', 'list')) - self.assertEquals(f5.func_code.co_varnames, - ('.0', 'two', 'compound', 'first')) - def a1(one_arg,): pass - def a2(two, args,): pass - def v0(*rest): pass - def v1(a, *rest): pass - def v2(a, b, *rest): pass - def v3(a, (b, c), *rest): return a, b, c, rest - - f1() - f2(1) - f2(1,) - f3(1, 2) - f3(1, 2,) - f4(1, (2, (3, 4))) - v0() - v0(1) - v0(1,) - v0(1,2) - v0(1,2,3,4,5,6,7,8,9,0) - v1(1) - v1(1,) - v1(1,2) - v1(1,2,3) - v1(1,2,3,4,5,6,7,8,9,0) - v2(1,2) - v2(1,2,3) - v2(1,2,3,4) - v2(1,2,3,4,5,6,7,8,9,0) - v3(1,(2,3)) - v3(1,(2,3),4) - v3(1,(2,3),4,5,6,7,8,9,0) - - # ceval unpacks the formal arguments into the first argcount names; - # thus, the names nested inside tuples must appear after these names. - if sys.platform.startswith('java'): - self.assertEquals(v3.func_code.co_varnames, ('a', '(b, c)', 'rest', 'b', 'c')) - else: - self.assertEquals(v3.func_code.co_varnames, ('a', '.1', 'rest', 'b', 'c')) - self.assertEquals(v3(1, (2, 3), 4), (1, 2, 3, (4,))) - def d01(a=1): pass - d01() - d01(1) - d01(*(1,)) - d01(**{'a':2}) - def d11(a, b=1): pass - d11(1) - d11(1, 2) - d11(1, **{'b':2}) - def d21(a, b, c=1): pass - d21(1, 2) - d21(1, 2, 3) - d21(*(1, 2, 3)) - d21(1, *(2, 3)) - d21(1, 2, *(3,)) - d21(1, 2, **{'c':3}) - def d02(a=1, b=2): pass - d02() - d02(1) - d02(1, 2) - d02(*(1, 2)) - d02(1, *(2,)) - d02(1, **{'b':2}) - d02(**{'a': 1, 'b': 2}) - def d12(a, b=1, c=2): pass - d12(1) - d12(1, 2) - d12(1, 2, 3) - def d22(a, b, c=1, d=2): pass - d22(1, 2) - d22(1, 2, 3) - d22(1, 2, 3, 4) - def d01v(a=1, *rest): pass - d01v() - d01v(1) - d01v(1, 2) - d01v(*(1, 2, 3, 4)) - d01v(*(1,)) - d01v(**{'a':2}) - def d11v(a, b=1, *rest): pass - d11v(1) - d11v(1, 2) - d11v(1, 2, 3) - def d21v(a, b, c=1, *rest): pass - d21v(1, 2) - d21v(1, 2, 3) - d21v(1, 2, 3, 4) - d21v(*(1, 2, 3, 4)) - d21v(1, 2, **{'c': 3}) - def d02v(a=1, b=2, *rest): pass - d02v() - d02v(1) - d02v(1, 2) - d02v(1, 2, 3) - d02v(1, *(2, 3, 4)) - d02v(**{'a': 1, 'b': 2}) - def d12v(a, b=1, c=2, *rest): pass - d12v(1) - d12v(1, 2) - d12v(1, 2, 3) - d12v(1, 2, 3, 4) - d12v(*(1, 2, 3, 4)) - d12v(1, 2, *(3, 4, 5)) - d12v(1, *(2,), **{'c': 3}) - def d22v(a, b, c=1, d=2, *rest): pass - d22v(1, 2) - d22v(1, 2, 3) - d22v(1, 2, 3, 4) - d22v(1, 2, 3, 4, 5) - d22v(*(1, 2, 3, 4)) - d22v(1, 2, *(3, 4, 5)) - d22v(1, *(2, 3), **{'d': 4}) - def d31v((x)): pass - d31v(1) - def d32v((x,)): pass - d32v((1,)) - - # keyword arguments after *arglist - def f(*args, **kwargs): - return args, kwargs - self.assertEquals(f(1, x=2, *[3, 4], y=5), ((1, 3, 4), - {'x':2, 'y':5})) - self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)") - self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)") - - # Check ast errors in *args and *kwargs - check_syntax_error(self, "f(*g(1=2))") - check_syntax_error(self, "f(**g(1=2))") - - def testLambdef(self): - ### lambdef: 'lambda' [varargslist] ':' test - l1 = lambda : 0 - self.assertEquals(l1(), 0) - l2 = lambda : a[d] # XXX just testing the expression - l3 = lambda : [2 < x for x in [-1, 3, 0L]] - self.assertEquals(l3(), [0, 1, 0]) - l4 = lambda x = lambda y = lambda z=1 : z : y() : x() - self.assertEquals(l4(), 1) - l5 = lambda x, y, z=2: x + y + z - self.assertEquals(l5(1, 2), 5) - self.assertEquals(l5(1, 2, 3), 6) - check_syntax_error(self, "lambda x: x = 2") - check_syntax_error(self, "lambda (None,): None") - - ### stmt: simple_stmt | compound_stmt - # Tested below - - def testSimpleStmt(self): - ### simple_stmt: small_stmt (';' small_stmt)* [';'] - x = 1; pass; del x - def foo(): - # verify statements that end with semi-colons - x = 1; pass; del x; - foo() - - ### small_stmt: expr_stmt | print_stmt | pass_stmt | del_stmt | flow_stmt | import_stmt | global_stmt | access_stmt | exec_stmt - # Tested below - - def testExprStmt(self): - # (exprlist '=')* exprlist - 1 - 1, 2, 3 - x = 1 - x = 1, 2, 3 - x = y = z = 1, 2, 3 - x, y, z = 1, 2, 3 - abc = a, b, c = x, y, z = xyz = 1, 2, (3, 4) - - check_syntax_error(self, "x + 1 = 1") - check_syntax_error(self, "a + 1 = b + 2") - - def testPrintStmt(self): - # 'print' (test ',')* [test] - import StringIO - - # Can't test printing to real stdout without comparing output - # which is not available in unittest. - save_stdout = sys.stdout - sys.stdout = StringIO.StringIO() - - print 1, 2, 3 - print 1, 2, 3, - print - print 0 or 1, 0 or 1, - print 0 or 1 - - # 'print' '>>' test ',' - print >> sys.stdout, 1, 2, 3 - print >> sys.stdout, 1, 2, 3, - print >> sys.stdout - print >> sys.stdout, 0 or 1, 0 or 1, - print >> sys.stdout, 0 or 1 - - # test printing to an instance - class Gulp: - def write(self, msg): pass - - gulp = Gulp() - print >> gulp, 1, 2, 3 - print >> gulp, 1, 2, 3, - print >> gulp - print >> gulp, 0 or 1, 0 or 1, - print >> gulp, 0 or 1 - - # test print >> None - def driver(): - oldstdout = sys.stdout - sys.stdout = Gulp() - try: - tellme(Gulp()) - tellme() - finally: - sys.stdout = oldstdout - - # we should see this once - def tellme(file=sys.stdout): - print >> file, 'hello world' - - driver() - - # we should not see this at all - def tellme(file=None): - print >> file, 'goodbye universe' - - driver() - - self.assertEqual(sys.stdout.getvalue(), '''\ -1 2 3 -1 2 3 -1 1 1 -1 2 3 -1 2 3 -1 1 1 -hello world -''') - sys.stdout = save_stdout - - # syntax errors - check_syntax_error(self, 'print ,') - check_syntax_error(self, 'print >> x,') - - def testDelStmt(self): - # 'del' exprlist - abc = [1,2,3] - x, y, z = abc - xyz = x, y, z - - del abc - del x, y, (z, xyz) - - def testPassStmt(self): - # 'pass' - pass - - # flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt - # Tested below - - def testBreakStmt(self): - # 'break' - while 1: break - - def testContinueStmt(self): - # 'continue' - i = 1 - while i: i = 0; continue - - msg = "" - while not msg: - msg = "ok" - try: - continue - msg = "continue failed to continue inside try" - except: - msg = "continue inside try called except block" - if msg != "ok": - self.fail(msg) - - msg = "" - while not msg: - msg = "finally block not called" - try: - continue - finally: - msg = "ok" - if msg != "ok": - self.fail(msg) - - def test_break_continue_loop(self): - # This test warrants an explanation. It is a test specifically for SF bugs - # #463359 and #462937. The bug is that a 'break' statement executed or - # exception raised inside a try/except inside a loop, *after* a continue - # statement has been executed in that loop, will cause the wrong number of - # arguments to be popped off the stack and the instruction pointer reset to - # a very small number (usually 0.) Because of this, the following test - # *must* written as a function, and the tracking vars *must* be function - # arguments with default values. Otherwise, the test will loop and loop. - - def test_inner(extra_burning_oil = 1, count=0): - big_hippo = 2 - while big_hippo: - count += 1 - try: - if extra_burning_oil and big_hippo == 1: - extra_burning_oil -= 1 - break - big_hippo -= 1 - continue - except: - raise - if count > 2 or big_hippo <> 1: - self.fail("continue then break in try/except in loop broken!") - test_inner() - - def testReturn(self): - # 'return' [testlist] - def g1(): return - def g2(): return 1 - g1() - x = g2() - check_syntax_error(self, "class foo:return 1") - - def testYield(self): - check_syntax_error(self, "class foo:yield 1") - - def testRaise(self): - # 'raise' test [',' test] - try: raise RuntimeError, 'just testing' - except RuntimeError: pass - try: raise KeyboardInterrupt - except KeyboardInterrupt: pass - - def testImport(self): - # 'import' dotted_as_names - import sys - import time, sys - # 'from' dotted_name 'import' ('*' | '(' import_as_names ')' | import_as_names) - from time import time - from time import (time) - # not testable inside a function, but already done at top of the module - # from sys import * - from sys import path, argv - from sys import (path, argv) - from sys import (path, argv,) - - def testGlobal(self): - # 'global' NAME (',' NAME)* - global a - global a, b - global one, two, three, four, five, six, seven, eight, nine, ten - - def testExec(self): - # 'exec' expr ['in' expr [',' expr]] - z = None - del z - exec 'z=1+1\n' - if z != 2: self.fail('exec \'z=1+1\'\\n') - del z - exec 'z=1+1' - if z != 2: self.fail('exec \'z=1+1\'') - z = None - del z - import types - if hasattr(types, "UnicodeType"): - exec r"""if 1: - exec u'z=1+1\n' - if z != 2: self.fail('exec u\'z=1+1\'\\n') - del z - exec u'z=1+1' - if z != 2: self.fail('exec u\'z=1+1\'')""" - g = {} - exec 'z = 1' in g - if g.has_key('__builtins__'): del g['__builtins__'] - if g != {'z': 1}: self.fail('exec \'z = 1\' in g') - g = {} - l = {} - - import warnings - warnings.filterwarnings("ignore", "global statement", module="") - exec 'global a; a = 1; b = 2' in g, l - if g.has_key('__builtins__'): del g['__builtins__'] - if l.has_key('__builtins__'): del l['__builtins__'] - if (g, l) != ({'a':1}, {'b':2}): - self.fail('exec ... in g (%s), l (%s)' %(g,l)) - - def testAssert(self): - # assert_stmt: 'assert' test [',' test] - assert 1 - assert 1, 1 - assert lambda x:x - assert 1, lambda x:x+1 - try: - assert 0, "msg" - except AssertionError, e: - self.assertEquals(e.args[0], "msg") - else: - if __debug__: - self.fail("AssertionError not raised by assert 0") - - ### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef - # Tested below - - def testIf(self): - # 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] - if 1: pass - if 1: pass - else: pass - if 0: pass - elif 0: pass - if 0: pass - elif 0: pass - elif 0: pass - elif 0: pass - else: pass - - def testWhile(self): - # 'while' test ':' suite ['else' ':' suite] - while 0: pass - while 0: pass - else: pass - - # Issue1920: "while 0" is optimized away, - # ensure that the "else" clause is still present. - x = 0 - while 0: - x = 1 - else: - x = 2 - self.assertEquals(x, 2) - - def testFor(self): - # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite] - for i in 1, 2, 3: pass - for i, j, k in (): pass - else: pass - class Squares: - def __init__(self, max): - self.max = max - self.sofar = [] - def __len__(self): return len(self.sofar) - def __getitem__(self, i): - if not 0 <= i < self.max: raise IndexError - n = len(self.sofar) - while n <= i: - self.sofar.append(n*n) - n = n+1 - return self.sofar[i] - n = 0 - for x in Squares(10): n = n+x - if n != 285: - self.fail('for over growing sequence') - - result = [] - for x, in [(1,), (2,), (3,)]: - result.append(x) - self.assertEqual(result, [1, 2, 3]) - - def testTry(self): - ### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite] - ### | 'try' ':' suite 'finally' ':' suite - ### except_clause: 'except' [expr [('as' | ',') expr]] - try: - 1/0 - except ZeroDivisionError: - pass - else: - pass - try: 1/0 - except EOFError: pass - except TypeError as msg: pass - except RuntimeError, msg: pass - except: pass - else: pass - try: 1/0 - except (EOFError, TypeError, ZeroDivisionError): pass - try: 1/0 - except (EOFError, TypeError, ZeroDivisionError), msg: pass - try: pass - finally: pass - - def testSuite(self): - # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT - if 1: pass - if 1: - pass - if 1: - # - # - # - pass - pass - # - pass - # - - def testTest(self): - ### and_test ('or' and_test)* - ### and_test: not_test ('and' not_test)* - ### not_test: 'not' not_test | comparison - if not 1: pass - if 1 and 1: pass - if 1 or 1: pass - if not not not 1: pass - if not 1 and 1 and 1: pass - if 1 and 1 or 1 and 1 and 1 or not 1 and 1: pass - - def testComparison(self): - ### comparison: expr (comp_op expr)* - ### comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' - if 1: pass - x = (1 == 1) - if 1 == 1: pass - if 1 != 1: pass - if 1 <> 1: pass - if 1 < 1: pass - if 1 > 1: pass - if 1 <= 1: pass - if 1 >= 1: pass - if 1 is 1: pass - if 1 is not 1: pass - if 1 in (): pass - if 1 not in (): pass - if 1 < 1 > 1 == 1 >= 1 <= 1 <> 1 != 1 in 1 not in 1 is 1 is not 1: pass - - def testBinaryMaskOps(self): - x = 1 & 1 - x = 1 ^ 1 - x = 1 | 1 - - def testShiftOps(self): - x = 1 << 1 - x = 1 >> 1 - x = 1 << 1 >> 1 - - def testAdditiveOps(self): - x = 1 - x = 1 + 1 - x = 1 - 1 - 1 - x = 1 - 1 + 1 - 1 + 1 - - def testMultiplicativeOps(self): - x = 1 * 1 - x = 1 / 1 - x = 1 % 1 - x = 1 / 1 * 1 % 1 - - def testUnaryOps(self): - x = +1 - x = -1 - x = ~1 - x = ~1 ^ 1 & 1 | 1 & 1 ^ -1 - x = -1*1/1 + 1*1 - ---1*1 - - def testSelectors(self): - ### trailer: '(' [testlist] ')' | '[' subscript ']' | '.' NAME - ### subscript: expr | [expr] ':' [expr] - - import sys, time - c = sys.path[0] - x = time.time() - x = sys.modules['time'].time() - a = '01234' - c = a[0] - c = a[-1] - s = a[0:5] - s = a[:5] - s = a[0:] - s = a[:] - s = a[-5:] - s = a[:-1] - s = a[-4:-3] - # A rough test of SF bug 1333982. https://bugs.python.org/issue1333982 - # The testing here is fairly incomplete. - # Test cases should include: commas with 1 and 2 colons - d = {} - d[1] = 1 - d[1,] = 2 - d[1,2] = 3 - d[1,2,3] = 4 - L = list(d) - L.sort() - self.assertEquals(str(L), '[1, (1,), (1, 2), (1, 2, 3)]') - - def testAtoms(self): - ### atom: '(' [testlist] ')' | '[' [testlist] ']' | '{' [dictmaker] '}' | '`' testlist '`' | NAME | NUMBER | STRING - ### dictmaker: test ':' test (',' test ':' test)* [','] - - x = (1) - x = (1 or 2 or 3) - x = (1 or 2 or 3, 2, 3) - - x = [] - x = [1] - x = [1 or 2 or 3] - x = [1 or 2 or 3, 2, 3] - x = [] - - x = {} - x = {'one': 1} - x = {'one': 1,} - x = {'one' or 'two': 1 or 2} - x = {'one': 1, 'two': 2} - x = {'one': 1, 'two': 2,} - x = {'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6} - - x = `x` - x = `1 or 2 or 3` - self.assertEqual(`1,2`, '(1, 2)') - - x = x - x = 'x' - x = 123 - - ### exprlist: expr (',' expr)* [','] - ### testlist: test (',' test)* [','] - # These have been exercised enough above - - def testClassdef(self): - # 'class' NAME ['(' [testlist] ')'] ':' suite - class B: pass - class B2(): pass - class C1(B): pass - class C2(B): pass - class D(C1, C2, B): pass - class C: - def meth1(self): pass - def meth2(self, arg): pass - def meth3(self, a1, a2): pass - # decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE - # decorators: decorator+ - # decorated: decorators (classdef | funcdef) - def class_decorator(x): - x.decorated = True - return x - @class_decorator - class G: - pass - self.assertEqual(G.decorated, True) - - def testListcomps(self): - # list comprehension tests - nums = [1, 2, 3, 4, 5] - strs = ["Apple", "Banana", "Coconut"] - spcs = [" Apple", " Banana ", "Coco nut "] - - self.assertEqual([s.strip() for s in spcs], ['Apple', 'Banana', 'Coco nut']) - self.assertEqual([3 * x for x in nums], [3, 6, 9, 12, 15]) - self.assertEqual([x for x in nums if x > 2], [3, 4, 5]) - self.assertEqual([(i, s) for i in nums for s in strs], - [(1, 'Apple'), (1, 'Banana'), (1, 'Coconut'), - (2, 'Apple'), (2, 'Banana'), (2, 'Coconut'), - (3, 'Apple'), (3, 'Banana'), (3, 'Coconut'), - (4, 'Apple'), (4, 'Banana'), (4, 'Coconut'), - (5, 'Apple'), (5, 'Banana'), (5, 'Coconut')]) - self.assertEqual([(i, s) for i in nums for s in [f for f in strs if "n" in f]], - [(1, 'Banana'), (1, 'Coconut'), (2, 'Banana'), (2, 'Coconut'), - (3, 'Banana'), (3, 'Coconut'), (4, 'Banana'), (4, 'Coconut'), - (5, 'Banana'), (5, 'Coconut')]) - self.assertEqual([(lambda a:[a**i for i in range(a+1)])(j) for j in range(5)], - [[1], [1, 1], [1, 2, 4], [1, 3, 9, 27], [1, 4, 16, 64, 256]]) - - def test_in_func(l): - return [None < x < 3 for x in l if x > 2] - - self.assertEqual(test_in_func(nums), [False, False, False]) - - def test_nested_front(): - self.assertEqual([[y for y in [x, x + 1]] for x in [1,3,5]], - [[1, 2], [3, 4], [5, 6]]) - - test_nested_front() - - check_syntax_error(self, "[i, s for i in nums for s in strs]") - check_syntax_error(self, "[x if y]") - - suppliers = [ - (1, "Boeing"), - (2, "Ford"), - (3, "Macdonalds") - ] - - parts = [ - (10, "Airliner"), - (20, "Engine"), - (30, "Cheeseburger") - ] - - suppart = [ - (1, 10), (1, 20), (2, 20), (3, 30) - ] - - x = [ - (sname, pname) - for (sno, sname) in suppliers - for (pno, pname) in parts - for (sp_sno, sp_pno) in suppart - if sno == sp_sno and pno == sp_pno - ] - - self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'), - ('Macdonalds', 'Cheeseburger')]) - - def testGenexps(self): - # generator expression tests - g = ([x for x in range(10)] for x in range(1)) - self.assertEqual(g.next(), [x for x in range(10)]) - try: - g.next() - self.fail('should produce StopIteration exception') - except StopIteration: - pass - - a = 1 - try: - g = (a for d in a) - g.next() - self.fail('should produce TypeError') - except TypeError: - pass - - self.assertEqual(list((x, y) for x in 'abcd' for y in 'abcd'), [(x, y) for x in 'abcd' for y in 'abcd']) - self.assertEqual(list((x, y) for x in 'ab' for y in 'xy'), [(x, y) for x in 'ab' for y in 'xy']) - - a = [x for x in range(10)] - b = (x for x in (y for y in a)) - self.assertEqual(sum(b), sum([x for x in range(10)])) - - self.assertEqual(sum(x**2 for x in range(10)), sum([x**2 for x in range(10)])) - self.assertEqual(sum(x*x for x in range(10) if x%2), sum([x*x for x in range(10) if x%2])) - self.assertEqual(sum(x for x in (y for y in range(10))), sum([x for x in range(10)])) - self.assertEqual(sum(x for x in (y for y in (z for z in range(10)))), sum([x for x in range(10)])) - self.assertEqual(sum(x for x in [y for y in (z for z in range(10))]), sum([x for x in range(10)])) - self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True)) if True), sum([x for x in range(10)])) - self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True) if False) if True), 0) - check_syntax_error(self, "foo(x for x in range(10), 100)") - check_syntax_error(self, "foo(100, x for x in range(10))") - - def testComprehensionSpecials(self): - # test for outmost iterable precomputation - x = 10; g = (i for i in range(x)); x = 5 - self.assertEqual(len(list(g)), 10) - - # This should hold, since we're only precomputing outmost iterable. - x = 10; t = False; g = ((i,j) for i in range(x) if t for j in range(x)) - x = 5; t = True; - self.assertEqual([(i,j) for i in range(10) for j in range(5)], list(g)) - - # Grammar allows multiple adjacent 'if's in listcomps and genexps, - # even though it's silly. Make sure it works (ifelse broke this.) - self.assertEqual([ x for x in range(10) if x % 2 if x % 3 ], [1, 5, 7]) - self.assertEqual(list(x for x in range(10) if x % 2 if x % 3), [1, 5, 7]) - - # verify unpacking single element tuples in listcomp/genexp. - self.assertEqual([x for x, in [(4,), (5,), (6,)]], [4, 5, 6]) - self.assertEqual(list(x for x, in [(7,), (8,), (9,)]), [7, 8, 9]) - - def test_with_statement(self): - class manager(object): - def __enter__(self): - return (1, 2) - def __exit__(self, *args): - pass - - with manager(): - pass - with manager() as x: - pass - with manager() as (x, y): - pass - with manager(), manager(): - pass - with manager() as x, manager() as y: - pass - with manager() as x, manager(): - pass - - def testIfElseExpr(self): - # Test ifelse expressions in various cases - def _checkeval(msg, ret): - "helper to check that evaluation of expressions is done correctly" - print x - return ret - - self.assertEqual([ x() for x in lambda: True, lambda: False if x() ], [True]) - self.assertEqual([ x() for x in (lambda: True, lambda: False) if x() ], [True]) - self.assertEqual([ x(False) for x in (lambda x: False if x else True, lambda x: True if x else False) if x(False) ], [True]) - self.assertEqual((5 if 1 else _checkeval("check 1", 0)), 5) - self.assertEqual((_checkeval("check 2", 0) if 0 else 5), 5) - self.assertEqual((5 and 6 if 0 else 1), 1) - self.assertEqual(((5 and 6) if 0 else 1), 1) - self.assertEqual((5 and (6 if 1 else 1)), 6) - self.assertEqual((0 or _checkeval("check 3", 2) if 0 else 3), 3) - self.assertEqual((1 or _checkeval("check 4", 2) if 1 else _checkeval("check 5", 3)), 1) - self.assertEqual((0 or 5 if 1 else _checkeval("check 6", 3)), 5) - self.assertEqual((not 5 if 1 else 1), False) - self.assertEqual((not 5 if 0 else 1), 1) - self.assertEqual((6 + 1 if 1 else 2), 7) - self.assertEqual((6 - 1 if 1 else 2), 5) - self.assertEqual((6 * 2 if 1 else 4), 12) - self.assertEqual((6 / 2 if 1 else 3), 3) - self.assertEqual((6 < 4 if 0 else 2), 2) - - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/test/test_lib2to3/data/py3_test_grammar.py b/Lib/test/test_lib2to3/data/py3_test_grammar.py deleted file mode 100644 index 774851f5bd7e85..00000000000000 --- a/Lib/test/test_lib2to3/data/py3_test_grammar.py +++ /dev/null @@ -1,956 +0,0 @@ -# Python test set -- part 1, grammar. -# This just tests whether the parser accepts them all. - -# NOTE: When you run this test as a script from the command line, you -# get warnings about certain hex/oct constants. Since those are -# issued by the parser, you can't suppress them by adding a -# filterwarnings() call to this module. Therefore, to shut up the -# regression test, the filterwarnings() call has been added to -# regrtest.py. - -from test.support import check_syntax_error -import unittest -import sys -# testing import * -from sys import * - -class TokenTests(unittest.TestCase): - - def testBackslash(self): - # Backslash means line continuation: - x = 1 \ - + 1 - self.assertEquals(x, 2, 'backslash for line continuation') - - # Backslash does not means continuation in comments :\ - x = 0 - self.assertEquals(x, 0, 'backslash ending comment') - - def testPlainIntegers(self): - self.assertEquals(type(000), type(0)) - self.assertEquals(0xff, 255) - self.assertEquals(0o377, 255) - self.assertEquals(2147483647, 0o17777777777) - self.assertEquals(0b1001, 9) - # "0x" is not a valid literal - self.assertRaises(SyntaxError, eval, "0x") - from sys import maxsize - if maxsize == 2147483647: - self.assertEquals(-2147483647-1, -0o20000000000) - # XXX -2147483648 - self.assert_(0o37777777777 > 0) - self.assert_(0xffffffff > 0) - self.assert_(0b1111111111111111111111111111111 > 0) - for s in ('2147483648', '0o40000000000', '0x100000000', - '0b10000000000000000000000000000000'): - try: - x = eval(s) - except OverflowError: - self.fail("OverflowError on huge integer literal %r" % s) - elif maxsize == 9223372036854775807: - self.assertEquals(-9223372036854775807-1, -0o1000000000000000000000) - self.assert_(0o1777777777777777777777 > 0) - self.assert_(0xffffffffffffffff > 0) - self.assert_(0b11111111111111111111111111111111111111111111111111111111111111 > 0) - for s in '9223372036854775808', '0o2000000000000000000000', \ - '0x10000000000000000', \ - '0b100000000000000000000000000000000000000000000000000000000000000': - try: - x = eval(s) - except OverflowError: - self.fail("OverflowError on huge integer literal %r" % s) - else: - self.fail('Weird maxsize value %r' % maxsize) - - def testLongIntegers(self): - x = 0 - x = 0xffffffffffffffff - x = 0Xffffffffffffffff - x = 0o77777777777777777 - x = 0O77777777777777777 - x = 123456789012345678901234567890 - x = 0b100000000000000000000000000000000000000000000000000000000000000000000 - x = 0B111111111111111111111111111111111111111111111111111111111111111111111 - - def testUnderscoresInNumbers(self): - # Integers - x = 1_0 - x = 123_456_7_89 - x = 0xabc_123_4_5 - x = 0X_abc_123 - x = 0B11_01 - x = 0b_11_01 - x = 0o45_67 - x = 0O_45_67 - - # Floats - x = 3_1.4 - x = 03_1.4 - x = 3_1. - x = .3_1 - x = 3.1_4 - x = 0_3.1_4 - x = 3e1_4 - x = 3_1e+4_1 - x = 3_1E-4_1 - - def testFloats(self): - x = 3.14 - x = 314. - x = 0.314 - # XXX x = 000.314 - x = .314 - x = 3e14 - x = 3E14 - x = 3e-14 - x = 3e+14 - x = 3.e14 - x = .3e14 - x = 3.1e4 - - def testStringLiterals(self): - x = ''; y = ""; self.assert_(len(x) == 0 and x == y) - x = '\''; y = "'"; self.assert_(len(x) == 1 and x == y and ord(x) == 39) - x = '"'; y = "\""; self.assert_(len(x) == 1 and x == y and ord(x) == 34) - x = "doesn't \"shrink\" does it" - y = 'doesn\'t "shrink" does it' - self.assert_(len(x) == 24 and x == y) - x = "does \"shrink\" doesn't it" - y = 'does "shrink" doesn\'t it' - self.assert_(len(x) == 24 and x == y) - x = """ -The "quick" -brown fox -jumps over -the 'lazy' dog. -""" - y = '\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n' - self.assertEquals(x, y) - y = ''' -The "quick" -brown fox -jumps over -the 'lazy' dog. -''' - self.assertEquals(x, y) - y = "\n\ -The \"quick\"\n\ -brown fox\n\ -jumps over\n\ -the 'lazy' dog.\n\ -" - self.assertEquals(x, y) - y = '\n\ -The \"quick\"\n\ -brown fox\n\ -jumps over\n\ -the \'lazy\' dog.\n\ -' - self.assertEquals(x, y) - x = rf"hello \{True}"; y = f"hello \\{True}" - self.assertEquals(x, y) - - def testEllipsis(self): - x = ... - self.assert_(x is Ellipsis) - self.assertRaises(SyntaxError, eval, ".. .") - -class GrammarTests(unittest.TestCase): - - # single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE - # XXX can't test in a script -- this rule is only used when interactive - - # file_input: (NEWLINE | stmt)* ENDMARKER - # Being tested as this very moment this very module - - # expr_input: testlist NEWLINE - # XXX Hard to test -- used only in calls to input() - - def testEvalInput(self): - # testlist ENDMARKER - x = eval('1, 0 or 1') - - def testFuncdef(self): - ### [decorators] 'def' NAME parameters ['->' test] ':' suite - ### decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE - ### decorators: decorator+ - ### parameters: '(' [typedargslist] ')' - ### typedargslist: ((tfpdef ['=' test] ',')* - ### ('*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef) - ### | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) - ### tfpdef: NAME [':' test] - ### varargslist: ((vfpdef ['=' test] ',')* - ### ('*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef) - ### | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) - ### vfpdef: NAME - def f1(): pass - f1() - f1(*()) - f1(*(), **{}) - def f2(one_argument): pass - def f3(two, arguments): pass - self.assertEquals(f2.__code__.co_varnames, ('one_argument',)) - self.assertEquals(f3.__code__.co_varnames, ('two', 'arguments')) - def a1(one_arg,): pass - def a2(two, args,): pass - def v0(*rest): pass - def v1(a, *rest): pass - def v2(a, b, *rest): pass - - f1() - f2(1) - f2(1,) - f3(1, 2) - f3(1, 2,) - v0() - v0(1) - v0(1,) - v0(1,2) - v0(1,2,3,4,5,6,7,8,9,0) - v1(1) - v1(1,) - v1(1,2) - v1(1,2,3) - v1(1,2,3,4,5,6,7,8,9,0) - v2(1,2) - v2(1,2,3) - v2(1,2,3,4) - v2(1,2,3,4,5,6,7,8,9,0) - - def d01(a=1): pass - d01() - d01(1) - d01(*(1,)) - d01(**{'a':2}) - def d11(a, b=1): pass - d11(1) - d11(1, 2) - d11(1, **{'b':2}) - def d21(a, b, c=1): pass - d21(1, 2) - d21(1, 2, 3) - d21(*(1, 2, 3)) - d21(1, *(2, 3)) - d21(1, 2, *(3,)) - d21(1, 2, **{'c':3}) - def d02(a=1, b=2): pass - d02() - d02(1) - d02(1, 2) - d02(*(1, 2)) - d02(1, *(2,)) - d02(1, **{'b':2}) - d02(**{'a': 1, 'b': 2}) - def d12(a, b=1, c=2): pass - d12(1) - d12(1, 2) - d12(1, 2, 3) - def d22(a, b, c=1, d=2): pass - d22(1, 2) - d22(1, 2, 3) - d22(1, 2, 3, 4) - def d01v(a=1, *rest): pass - d01v() - d01v(1) - d01v(1, 2) - d01v(*(1, 2, 3, 4)) - d01v(*(1,)) - d01v(**{'a':2}) - def d11v(a, b=1, *rest): pass - d11v(1) - d11v(1, 2) - d11v(1, 2, 3) - def d21v(a, b, c=1, *rest): pass - d21v(1, 2) - d21v(1, 2, 3) - d21v(1, 2, 3, 4) - d21v(*(1, 2, 3, 4)) - d21v(1, 2, **{'c': 3}) - def d02v(a=1, b=2, *rest): pass - d02v() - d02v(1) - d02v(1, 2) - d02v(1, 2, 3) - d02v(1, *(2, 3, 4)) - d02v(**{'a': 1, 'b': 2}) - def d12v(a, b=1, c=2, *rest): pass - d12v(1) - d12v(1, 2) - d12v(1, 2, 3) - d12v(1, 2, 3, 4) - d12v(*(1, 2, 3, 4)) - d12v(1, 2, *(3, 4, 5)) - d12v(1, *(2,), **{'c': 3}) - def d22v(a, b, c=1, d=2, *rest): pass - d22v(1, 2) - d22v(1, 2, 3) - d22v(1, 2, 3, 4) - d22v(1, 2, 3, 4, 5) - d22v(*(1, 2, 3, 4)) - d22v(1, 2, *(3, 4, 5)) - d22v(1, *(2, 3), **{'d': 4}) - - # keyword argument type tests - try: - str('x', **{b'foo':1 }) - except TypeError: - pass - else: - self.fail('Bytes should not work as keyword argument names') - # keyword only argument tests - def pos0key1(*, key): return key - pos0key1(key=100) - def pos2key2(p1, p2, *, k1, k2=100): return p1,p2,k1,k2 - pos2key2(1, 2, k1=100) - pos2key2(1, 2, k1=100, k2=200) - pos2key2(1, 2, k2=100, k1=200) - def pos2key2dict(p1, p2, *, k1=100, k2, **kwarg): return p1,p2,k1,k2,kwarg - pos2key2dict(1,2,k2=100,tokwarg1=100,tokwarg2=200) - pos2key2dict(1,2,tokwarg1=100,tokwarg2=200, k2=100) - - # keyword arguments after *arglist - def f(*args, **kwargs): - return args, kwargs - self.assertEquals(f(1, x=2, *[3, 4], y=5), ((1, 3, 4), - {'x':2, 'y':5})) - self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)") - self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)") - - # argument annotation tests - def f(x) -> list: pass - self.assertEquals(f.__annotations__, {'return': list}) - def f(x:int): pass - self.assertEquals(f.__annotations__, {'x': int}) - def f(*x:str): pass - self.assertEquals(f.__annotations__, {'x': str}) - def f(**x:float): pass - self.assertEquals(f.__annotations__, {'x': float}) - def f(x, y:1+2): pass - self.assertEquals(f.__annotations__, {'y': 3}) - def f(a, b:1, c:2, d): pass - self.assertEquals(f.__annotations__, {'b': 1, 'c': 2}) - def f(a, b:1, c:2, d, e:3=4, f=5, *g:6): pass - self.assertEquals(f.__annotations__, - {'b': 1, 'c': 2, 'e': 3, 'g': 6}) - def f(a, b:1, c:2, d, e:3=4, f=5, *g:6, h:7, i=8, j:9=10, - **k:11) -> 12: pass - self.assertEquals(f.__annotations__, - {'b': 1, 'c': 2, 'e': 3, 'g': 6, 'h': 7, 'j': 9, - 'k': 11, 'return': 12}) - # Check for SF Bug #1697248 - mixing decorators and a return annotation - def null(x): return x - @null - def f(x) -> list: pass - self.assertEquals(f.__annotations__, {'return': list}) - - # test closures with a variety of oparg's - closure = 1 - def f(): return closure - def f(x=1): return closure - def f(*, k=1): return closure - def f() -> int: return closure - - # Check ast errors in *args and *kwargs - check_syntax_error(self, "f(*g(1=2))") - check_syntax_error(self, "f(**g(1=2))") - - def testLambdef(self): - ### lambdef: 'lambda' [varargslist] ':' test - l1 = lambda : 0 - self.assertEquals(l1(), 0) - l2 = lambda : a[d] # XXX just testing the expression - l3 = lambda : [2 < x for x in [-1, 3, 0]] - self.assertEquals(l3(), [0, 1, 0]) - l4 = lambda x = lambda y = lambda z=1 : z : y() : x() - self.assertEquals(l4(), 1) - l5 = lambda x, y, z=2: x + y + z - self.assertEquals(l5(1, 2), 5) - self.assertEquals(l5(1, 2, 3), 6) - check_syntax_error(self, "lambda x: x = 2") - check_syntax_error(self, "lambda (None,): None") - l6 = lambda x, y, *, k=20: x+y+k - self.assertEquals(l6(1,2), 1+2+20) - self.assertEquals(l6(1,2,k=10), 1+2+10) - - - ### stmt: simple_stmt | compound_stmt - # Tested below - - def testSimpleStmt(self): - ### simple_stmt: small_stmt (';' small_stmt)* [';'] - x = 1; pass; del x - def foo(): - # verify statements that end with semi-colons - x = 1; pass; del x; - foo() - - ### small_stmt: expr_stmt | pass_stmt | del_stmt | flow_stmt | import_stmt | global_stmt | access_stmt - # Tested below - - def testExprStmt(self): - # (exprlist '=')* exprlist - 1 - 1, 2, 3 - x = 1 - x = 1, 2, 3 - x = y = z = 1, 2, 3 - x, y, z = 1, 2, 3 - abc = a, b, c = x, y, z = xyz = 1, 2, (3, 4) - - check_syntax_error(self, "x + 1 = 1") - check_syntax_error(self, "a + 1 = b + 2") - - def testDelStmt(self): - # 'del' exprlist - abc = [1,2,3] - x, y, z = abc - xyz = x, y, z - - del abc - del x, y, (z, xyz) - - def testPassStmt(self): - # 'pass' - pass - - # flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt - # Tested below - - def testBreakStmt(self): - # 'break' - while 1: break - - def testContinueStmt(self): - # 'continue' - i = 1 - while i: i = 0; continue - - msg = "" - while not msg: - msg = "ok" - try: - continue - msg = "continue failed to continue inside try" - except: - msg = "continue inside try called except block" - if msg != "ok": - self.fail(msg) - - msg = "" - while not msg: - msg = "finally block not called" - try: - continue - finally: - msg = "ok" - if msg != "ok": - self.fail(msg) - - def test_break_continue_loop(self): - # This test warrants an explanation. It is a test specifically for SF bugs - # #463359 and #462937. The bug is that a 'break' statement executed or - # exception raised inside a try/except inside a loop, *after* a continue - # statement has been executed in that loop, will cause the wrong number of - # arguments to be popped off the stack and the instruction pointer reset to - # a very small number (usually 0.) Because of this, the following test - # *must* written as a function, and the tracking vars *must* be function - # arguments with default values. Otherwise, the test will loop and loop. - - def test_inner(extra_burning_oil = 1, count=0): - big_hippo = 2 - while big_hippo: - count += 1 - try: - if extra_burning_oil and big_hippo == 1: - extra_burning_oil -= 1 - break - big_hippo -= 1 - continue - except: - raise - if count > 2 or big_hippo != 1: - self.fail("continue then break in try/except in loop broken!") - test_inner() - - def testReturn(self): - # 'return' [testlist_star_expr] - def g1(): return - def g2(): return 1 - return_list = [2, 3] - def g3(): return 1, *return_list - g1() - x = g2() - x3 = g3() - check_syntax_error(self, "class foo:return 1") - - def testYield(self): - # 'yield' [yield_arg] - def g1(): yield 1 - yield_list = [2, 3] - def g2(): yield 1, *yield_list - def g3(): yield from iter(yield_list) - x1 = g1() - x2 = g2() - x3 = g3() - check_syntax_error(self, "class foo:yield 1") - check_syntax_error(self, "def g4(): yield from *a") - - def testRaise(self): - # 'raise' test [',' test] - try: raise RuntimeError('just testing') - except RuntimeError: pass - try: raise KeyboardInterrupt - except KeyboardInterrupt: pass - - def testImport(self): - # 'import' dotted_as_names - import sys - import time, sys - # 'from' dotted_name 'import' ('*' | '(' import_as_names ')' | import_as_names) - from time import time - from time import (time) - # not testable inside a function, but already done at top of the module - # from sys import * - from sys import path, argv - from sys import (path, argv) - from sys import (path, argv,) - - def testGlobal(self): - # 'global' NAME (',' NAME)* - global a - global a, b - global one, two, three, four, five, six, seven, eight, nine, ten - - def testNonlocal(self): - # 'nonlocal' NAME (',' NAME)* - x = 0 - y = 0 - def f(): - nonlocal x - nonlocal x, y - - def testAssert(self): - # assert_stmt: 'assert' test [',' test] - assert 1 - assert 1, 1 - assert lambda x:x - assert 1, lambda x:x+1 - try: - assert 0, "msg" - except AssertionError as e: - self.assertEquals(e.args[0], "msg") - else: - if __debug__: - self.fail("AssertionError not raised by assert 0") - - ### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef - # Tested below - - def testIf(self): - # 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] - if 1: pass - if 1: pass - else: pass - if 0: pass - elif 0: pass - if 0: pass - elif 0: pass - elif 0: pass - elif 0: pass - else: pass - - def testWhile(self): - # 'while' test ':' suite ['else' ':' suite] - while 0: pass - while 0: pass - else: pass - - # Issue1920: "while 0" is optimized away, - # ensure that the "else" clause is still present. - x = 0 - while 0: - x = 1 - else: - x = 2 - self.assertEquals(x, 2) - - def testFor(self): - # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite] - for i in 1, 2, 3: pass - for i, j, k in (): pass - else: pass - class Squares: - def __init__(self, max): - self.max = max - self.sofar = [] - def __len__(self): return len(self.sofar) - def __getitem__(self, i): - if not 0 <= i < self.max: raise IndexError - n = len(self.sofar) - while n <= i: - self.sofar.append(n*n) - n = n+1 - return self.sofar[i] - n = 0 - for x in Squares(10): n = n+x - if n != 285: - self.fail('for over growing sequence') - - result = [] - for x, in [(1,), (2,), (3,)]: - result.append(x) - self.assertEqual(result, [1, 2, 3]) - - def testTry(self): - ### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite] - ### | 'try' ':' suite 'finally' ':' suite - ### except_clause: 'except' [expr ['as' expr]] - try: - 1/0 - except ZeroDivisionError: - pass - else: - pass - try: 1/0 - except EOFError: pass - except TypeError as msg: pass - except RuntimeError as msg: pass - except: pass - else: pass - try: 1/0 - except (EOFError, TypeError, ZeroDivisionError): pass - try: 1/0 - except (EOFError, TypeError, ZeroDivisionError) as msg: pass - try: pass - finally: pass - - def testSuite(self): - # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT - if 1: pass - if 1: - pass - if 1: - # - # - # - pass - pass - # - pass - # - - def testTest(self): - ### and_test ('or' and_test)* - ### and_test: not_test ('and' not_test)* - ### not_test: 'not' not_test | comparison - if not 1: pass - if 1 and 1: pass - if 1 or 1: pass - if not not not 1: pass - if not 1 and 1 and 1: pass - if 1 and 1 or 1 and 1 and 1 or not 1 and 1: pass - - def testComparison(self): - ### comparison: expr (comp_op expr)* - ### comp_op: '<'|'>'|'=='|'>='|'<='|'!='|'in'|'not' 'in'|'is'|'is' 'not' - if 1: pass - x = (1 == 1) - if 1 == 1: pass - if 1 != 1: pass - if 1 < 1: pass - if 1 > 1: pass - if 1 <= 1: pass - if 1 >= 1: pass - if 1 is 1: pass - if 1 is not 1: pass - if 1 in (): pass - if 1 not in (): pass - if 1 < 1 > 1 == 1 >= 1 <= 1 != 1 in 1 not in 1 is 1 is not 1: pass - - def testBinaryMaskOps(self): - x = 1 & 1 - x = 1 ^ 1 - x = 1 | 1 - - def testShiftOps(self): - x = 1 << 1 - x = 1 >> 1 - x = 1 << 1 >> 1 - - def testAdditiveOps(self): - x = 1 - x = 1 + 1 - x = 1 - 1 - 1 - x = 1 - 1 + 1 - 1 + 1 - - def testMultiplicativeOps(self): - x = 1 * 1 - x = 1 / 1 - x = 1 % 1 - x = 1 / 1 * 1 % 1 - - def testUnaryOps(self): - x = +1 - x = -1 - x = ~1 - x = ~1 ^ 1 & 1 | 1 & 1 ^ -1 - x = -1*1/1 + 1*1 - ---1*1 - - def testSelectors(self): - ### trailer: '(' [testlist] ')' | '[' subscript ']' | '.' NAME - ### subscript: expr | [expr] ':' [expr] - - import sys, time - c = sys.path[0] - x = time.time() - x = sys.modules['time'].time() - a = '01234' - c = a[0] - c = a[-1] - s = a[0:5] - s = a[:5] - s = a[0:] - s = a[:] - s = a[-5:] - s = a[:-1] - s = a[-4:-3] - # A rough test of SF bug 1333982. https://bugs.python.org/issue1333982 - # The testing here is fairly incomplete. - # Test cases should include: commas with 1 and 2 colons - d = {} - d[1] = 1 - d[1,] = 2 - d[1,2] = 3 - d[1,2,3] = 4 - L = list(d) - L.sort(key=lambda x: x if isinstance(x, tuple) else ()) - self.assertEquals(str(L), '[1, (1,), (1, 2), (1, 2, 3)]') - - def testAtoms(self): - ### atom: '(' [testlist] ')' | '[' [testlist] ']' | '{' [dictsetmaker] '}' | NAME | NUMBER | STRING - ### dictsetmaker: (test ':' test (',' test ':' test)* [',']) | (test (',' test)* [',']) - - x = (1) - x = (1 or 2 or 3) - x = (1 or 2 or 3, 2, 3) - - x = [] - x = [1] - x = [1 or 2 or 3] - x = [1 or 2 or 3, 2, 3] - x = [] - - x = {} - x = {'one': 1} - x = {'one': 1,} - x = {'one' or 'two': 1 or 2} - x = {'one': 1, 'two': 2} - x = {'one': 1, 'two': 2,} - x = {'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6} - - x = {'one'} - x = {'one', 1,} - x = {'one', 'two', 'three'} - x = {2, 3, 4,} - - x = x - x = 'x' - x = 123 - - ### exprlist: expr (',' expr)* [','] - ### testlist: test (',' test)* [','] - # These have been exercised enough above - - def testClassdef(self): - # 'class' NAME ['(' [testlist] ')'] ':' suite - class B: pass - class B2(): pass - class C1(B): pass - class C2(B): pass - class D(C1, C2, B): pass - class C: - def meth1(self): pass - def meth2(self, arg): pass - def meth3(self, a1, a2): pass - - # decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE - # decorators: decorator+ - # decorated: decorators (classdef | funcdef) - def class_decorator(x): return x - @class_decorator - class G: pass - - def testDictcomps(self): - # dictorsetmaker: ( (test ':' test (comp_for | - # (',' test ':' test)* [','])) | - # (test (comp_for | (',' test)* [','])) ) - nums = [1, 2, 3] - self.assertEqual({i:i+1 for i in nums}, {1: 2, 2: 3, 3: 4}) - - def testListcomps(self): - # list comprehension tests - nums = [1, 2, 3, 4, 5] - strs = ["Apple", "Banana", "Coconut"] - spcs = [" Apple", " Banana ", "Coco nut "] - - self.assertEqual([s.strip() for s in spcs], ['Apple', 'Banana', 'Coco nut']) - self.assertEqual([3 * x for x in nums], [3, 6, 9, 12, 15]) - self.assertEqual([x for x in nums if x > 2], [3, 4, 5]) - self.assertEqual([(i, s) for i in nums for s in strs], - [(1, 'Apple'), (1, 'Banana'), (1, 'Coconut'), - (2, 'Apple'), (2, 'Banana'), (2, 'Coconut'), - (3, 'Apple'), (3, 'Banana'), (3, 'Coconut'), - (4, 'Apple'), (4, 'Banana'), (4, 'Coconut'), - (5, 'Apple'), (5, 'Banana'), (5, 'Coconut')]) - self.assertEqual([(i, s) for i in nums for s in [f for f in strs if "n" in f]], - [(1, 'Banana'), (1, 'Coconut'), (2, 'Banana'), (2, 'Coconut'), - (3, 'Banana'), (3, 'Coconut'), (4, 'Banana'), (4, 'Coconut'), - (5, 'Banana'), (5, 'Coconut')]) - self.assertEqual([(lambda a:[a**i for i in range(a+1)])(j) for j in range(5)], - [[1], [1, 1], [1, 2, 4], [1, 3, 9, 27], [1, 4, 16, 64, 256]]) - - def test_in_func(l): - return [0 < x < 3 for x in l if x > 2] - - self.assertEqual(test_in_func(nums), [False, False, False]) - - def test_nested_front(): - self.assertEqual([[y for y in [x, x + 1]] for x in [1,3,5]], - [[1, 2], [3, 4], [5, 6]]) - - test_nested_front() - - check_syntax_error(self, "[i, s for i in nums for s in strs]") - check_syntax_error(self, "[x if y]") - - suppliers = [ - (1, "Boeing"), - (2, "Ford"), - (3, "Macdonalds") - ] - - parts = [ - (10, "Airliner"), - (20, "Engine"), - (30, "Cheeseburger") - ] - - suppart = [ - (1, 10), (1, 20), (2, 20), (3, 30) - ] - - x = [ - (sname, pname) - for (sno, sname) in suppliers - for (pno, pname) in parts - for (sp_sno, sp_pno) in suppart - if sno == sp_sno and pno == sp_pno - ] - - self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'), - ('Macdonalds', 'Cheeseburger')]) - - def testGenexps(self): - # generator expression tests - g = ([x for x in range(10)] for x in range(1)) - self.assertEqual(next(g), [x for x in range(10)]) - try: - next(g) - self.fail('should produce StopIteration exception') - except StopIteration: - pass - - a = 1 - try: - g = (a for d in a) - next(g) - self.fail('should produce TypeError') - except TypeError: - pass - - self.assertEqual(list((x, y) for x in 'abcd' for y in 'abcd'), [(x, y) for x in 'abcd' for y in 'abcd']) - self.assertEqual(list((x, y) for x in 'ab' for y in 'xy'), [(x, y) for x in 'ab' for y in 'xy']) - - a = [x for x in range(10)] - b = (x for x in (y for y in a)) - self.assertEqual(sum(b), sum([x for x in range(10)])) - - self.assertEqual(sum(x**2 for x in range(10)), sum([x**2 for x in range(10)])) - self.assertEqual(sum(x*x for x in range(10) if x%2), sum([x*x for x in range(10) if x%2])) - self.assertEqual(sum(x for x in (y for y in range(10))), sum([x for x in range(10)])) - self.assertEqual(sum(x for x in (y for y in (z for z in range(10)))), sum([x for x in range(10)])) - self.assertEqual(sum(x for x in [y for y in (z for z in range(10))]), sum([x for x in range(10)])) - self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True)) if True), sum([x for x in range(10)])) - self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True) if False) if True), 0) - check_syntax_error(self, "foo(x for x in range(10), 100)") - check_syntax_error(self, "foo(100, x for x in range(10))") - - def testComprehensionSpecials(self): - # test for outmost iterable precomputation - x = 10; g = (i for i in range(x)); x = 5 - self.assertEqual(len(list(g)), 10) - - # This should hold, since we're only precomputing outmost iterable. - x = 10; t = False; g = ((i,j) for i in range(x) if t for j in range(x)) - x = 5; t = True; - self.assertEqual([(i,j) for i in range(10) for j in range(5)], list(g)) - - # Grammar allows multiple adjacent 'if's in listcomps and genexps, - # even though it's silly. Make sure it works (ifelse broke this.) - self.assertEqual([ x for x in range(10) if x % 2 if x % 3 ], [1, 5, 7]) - self.assertEqual(list(x for x in range(10) if x % 2 if x % 3), [1, 5, 7]) - - # verify unpacking single element tuples in listcomp/genexp. - self.assertEqual([x for x, in [(4,), (5,), (6,)]], [4, 5, 6]) - self.assertEqual(list(x for x, in [(7,), (8,), (9,)]), [7, 8, 9]) - - def test_with_statement(self): - class manager(object): - def __enter__(self): - return (1, 2) - def __exit__(self, *args): - pass - - with manager(): - pass - with manager() as x: - pass - with manager() as (x, y): - pass - with manager(), manager(): - pass - with manager() as x, manager() as y: - pass - with manager() as x, manager(): - pass - - def testIfElseExpr(self): - # Test ifelse expressions in various cases - def _checkeval(msg, ret): - "helper to check that evaluation of expressions is done correctly" - print(x) - return ret - - # the next line is not allowed anymore - #self.assertEqual([ x() for x in lambda: True, lambda: False if x() ], [True]) - self.assertEqual([ x() for x in (lambda: True, lambda: False) if x() ], [True]) - self.assertEqual([ x(False) for x in (lambda x: False if x else True, lambda x: True if x else False) if x(False) ], [True]) - self.assertEqual((5 if 1 else _checkeval("check 1", 0)), 5) - self.assertEqual((_checkeval("check 2", 0) if 0 else 5), 5) - self.assertEqual((5 and 6 if 0 else 1), 1) - self.assertEqual(((5 and 6) if 0 else 1), 1) - self.assertEqual((5 and (6 if 1 else 1)), 6) - self.assertEqual((0 or _checkeval("check 3", 2) if 0 else 3), 3) - self.assertEqual((1 or _checkeval("check 4", 2) if 1 else _checkeval("check 5", 3)), 1) - self.assertEqual((0 or 5 if 1 else _checkeval("check 6", 3)), 5) - self.assertEqual((not 5 if 1 else 1), False) - self.assertEqual((not 5 if 0 else 1), 1) - self.assertEqual((6 + 1 if 1 else 2), 7) - self.assertEqual((6 - 1 if 1 else 2), 5) - self.assertEqual((6 * 2 if 1 else 4), 12) - self.assertEqual((6 / 2 if 1 else 3), 3) - self.assertEqual((6 < 4 if 0 else 2), 2) - - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/test/test_lib2to3/pytree_idempotency.py b/Lib/test/test_lib2to3/pytree_idempotency.py deleted file mode 100755 index eb2e2aa02ae0ed..00000000000000 --- a/Lib/test/test_lib2to3/pytree_idempotency.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Main program for testing the infrastructure.""" - -from __future__ import print_function - -__author__ = "Guido van Rossum " - -# Support imports (need to be imported first) -from . import support - -# Python imports -import os -import sys -import logging - -# Local imports -from lib2to3 import pytree -from lib2to3 import pgen2 -from lib2to3.pgen2 import driver - -logging.basicConfig() - -def main(): - gr = driver.load_grammar("Grammar.txt") - dr = driver.Driver(gr, convert=pytree.convert) - - fn = "example.py" - tree = dr.parse_file(fn, debug=True) - if not diff(fn, tree): - print("No diffs.") - if not sys.argv[1:]: - return # Pass a dummy argument to run the complete test suite below - - problems = [] - - # Process every imported module - for name in sys.modules: - mod = sys.modules[name] - if mod is None or not hasattr(mod, "__file__"): - continue - fn = mod.__file__ - if fn.endswith(".pyc"): - fn = fn[:-1] - if not fn.endswith(".py"): - continue - print("Parsing", fn, file=sys.stderr) - tree = dr.parse_file(fn, debug=True) - if diff(fn, tree): - problems.append(fn) - - # Process every single module on sys.path (but not in packages) - for dir in sys.path: - try: - names = os.listdir(dir) - except OSError: - continue - print("Scanning", dir, "...", file=sys.stderr) - for name in names: - if not name.endswith(".py"): - continue - print("Parsing", name, file=sys.stderr) - fn = os.path.join(dir, name) - try: - tree = dr.parse_file(fn, debug=True) - except pgen2.parse.ParseError as err: - print("ParseError:", err) - else: - if diff(fn, tree): - problems.append(fn) - - # Show summary of problem files - if not problems: - print("No problems. Congratulations!") - else: - print("Problems in following files:") - for fn in problems: - print("***", fn) - -def diff(fn, tree): - f = open("@", "w") - try: - f.write(str(tree)) - finally: - f.close() - try: - return os.system("diff -u %s @" % fn) - finally: - os.remove("@") - -if __name__ == "__main__": - main() diff --git a/Lib/test/test_lib2to3/support.py b/Lib/test/test_lib2to3/support.py deleted file mode 100644 index 9e56273e95992b..00000000000000 --- a/Lib/test/test_lib2to3/support.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Support code for test_*.py files""" -# Author: Collin Winter - -# Python imports -import unittest -import os -import os.path -from textwrap import dedent - -# Local imports -import lib2to3 -from lib2to3 import pytree, refactor -from lib2to3.pgen2 import driver as pgen2_driver - -lib2to3_dir = os.path.dirname(lib2to3.__file__) -test_dir = os.path.dirname(__file__) -proj_dir = os.path.normpath(os.path.join(test_dir, "..")) -grammar_path = os.path.join(lib2to3_dir, "Grammar.txt") -grammar = pgen2_driver.load_grammar(grammar_path) -grammar_no_print_statement = pgen2_driver.load_grammar(grammar_path) -del grammar_no_print_statement.keywords["print"] -driver = pgen2_driver.Driver(grammar, convert=pytree.convert) -driver_no_print_statement = pgen2_driver.Driver( - grammar_no_print_statement, - convert=pytree.convert -) - -def parse_string(string): - return driver.parse_string(reformat(string), debug=True) - -def run_all_tests(test_mod=None, tests=None): - if tests is None: - tests = unittest.TestLoader().loadTestsFromModule(test_mod) - unittest.TextTestRunner(verbosity=2).run(tests) - -def reformat(string): - return dedent(string) + "\n\n" - -def get_refactorer(fixer_pkg="lib2to3", fixers=None, options=None): - """ - A convenience function for creating a RefactoringTool for tests. - - fixers is a list of fixers for the RefactoringTool to use. By default - "lib2to3.fixes.*" is used. options is an optional dictionary of options to - be passed to the RefactoringTool. - """ - if fixers is not None: - fixers = [fixer_pkg + ".fixes.fix_" + fix for fix in fixers] - else: - fixers = refactor.get_fixers_from_package(fixer_pkg + ".fixes") - options = options or {} - return refactor.RefactoringTool(fixers, options, explicit=True) - -def _all_project_files(root, files): - for dirpath, dirnames, filenames in os.walk(root): - for filename in filenames: - if not filename.endswith(".py"): - continue - files.append(os.path.join(dirpath, filename)) - -def all_project_files(): - files = [] - _all_project_files(lib2to3_dir, files) - _all_project_files(test_dir, files) - # Sort to get more reproducible tests - files.sort() - return files - -TestCase = unittest.TestCase diff --git a/Lib/test/test_lib2to3/test_all_fixers.py b/Lib/test/test_lib2to3/test_all_fixers.py deleted file mode 100644 index d0fca7072482cb..00000000000000 --- a/Lib/test/test_lib2to3/test_all_fixers.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Tests that run all fixer modules over an input stream. - -This has been broken out into its own test module because of its -running time. -""" -# Author: Collin Winter - -# Python imports -import os.path -import test.support -import unittest - -# Local imports -from . import support - - -@test.support.requires_resource('cpu') -class Test_all(support.TestCase): - - def setUp(self): - self.refactor = support.get_refactorer() - - def refactor_file(self, filepath): - if test.support.verbose: - print(f"Refactor file: {filepath}") - if os.path.basename(filepath) == 'infinite_recursion.py': - # bpo-46542: Processing infinite_recursion.py can crash Python - # if Python is built in debug mode: lower the recursion limit - # to prevent a crash. - with test.support.infinite_recursion(150): - self.refactor.refactor_file(filepath) - else: - self.refactor.refactor_file(filepath) - - def test_all_project_files(self): - for filepath in support.all_project_files(): - with self.subTest(filepath=filepath): - self.refactor_file(filepath) - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/test/test_lib2to3/test_fixers.py b/Lib/test/test_lib2to3/test_fixers.py deleted file mode 100644 index 68efeee7abb4d5..00000000000000 --- a/Lib/test/test_lib2to3/test_fixers.py +++ /dev/null @@ -1,4649 +0,0 @@ -""" Test suite for the fixer modules """ - -# Python imports -import os -from itertools import chain -from operator import itemgetter - -# Local imports -from lib2to3 import pygram, fixer_util -from test.test_lib2to3 import support - - -class FixerTestCase(support.TestCase): - - # Other test cases can subclass this class and replace "fixer_pkg" with - # their own. - def setUp(self, fix_list=None, fixer_pkg="lib2to3", options=None): - if fix_list is None: - fix_list = [self.fixer] - self.refactor = support.get_refactorer(fixer_pkg, fix_list, options) - self.fixer_log = [] - self.filename = "" - - for fixer in chain(self.refactor.pre_order, - self.refactor.post_order): - fixer.log = self.fixer_log - - def _check(self, before, after): - before = support.reformat(before) - after = support.reformat(after) - tree = self.refactor.refactor_string(before, self.filename) - self.assertEqual(after, str(tree)) - return tree - - def check(self, before, after, ignore_warnings=False): - tree = self._check(before, after) - self.assertTrue(tree.was_changed) - if not ignore_warnings: - self.assertEqual(self.fixer_log, []) - - def warns(self, before, after, message, unchanged=False): - tree = self._check(before, after) - self.assertIn(message, "".join(self.fixer_log)) - if not unchanged: - self.assertTrue(tree.was_changed) - - def warns_unchanged(self, before, message): - self.warns(before, before, message, unchanged=True) - - def unchanged(self, before, ignore_warnings=False): - self._check(before, before) - if not ignore_warnings: - self.assertEqual(self.fixer_log, []) - - def assert_runs_after(self, *names): - fixes = [self.fixer] - fixes.extend(names) - r = support.get_refactorer("lib2to3", fixes) - (pre, post) = r.get_fixers() - n = "fix_" + self.fixer - if post and post[-1].__class__.__module__.endswith(n): - # We're the last fixer to run - return - if pre and pre[-1].__class__.__module__.endswith(n) and not post: - # We're the last in pre and post is empty - return - self.fail("Fixer run order (%s) is incorrect; %s should be last."\ - %(", ".join([x.__class__.__module__ for x in (pre+post)]), n)) - -class Test_ne(FixerTestCase): - fixer = "ne" - - def test_basic(self): - b = """if x <> y: - pass""" - - a = """if x != y: - pass""" - self.check(b, a) - - def test_no_spaces(self): - b = """if x<>y: - pass""" - - a = """if x!=y: - pass""" - self.check(b, a) - - def test_chained(self): - b = """if x<>y<>z: - pass""" - - a = """if x!=y!=z: - pass""" - self.check(b, a) - -class Test_has_key(FixerTestCase): - fixer = "has_key" - - def test_1(self): - b = """x = d.has_key("x") or d.has_key("y")""" - a = """x = "x" in d or "y" in d""" - self.check(b, a) - - def test_2(self): - b = """x = a.b.c.d.has_key("x") ** 3""" - a = """x = ("x" in a.b.c.d) ** 3""" - self.check(b, a) - - def test_3(self): - b = """x = a.b.has_key(1 + 2).__repr__()""" - a = """x = (1 + 2 in a.b).__repr__()""" - self.check(b, a) - - def test_4(self): - b = """x = a.b.has_key(1 + 2).__repr__() ** -3 ** 4""" - a = """x = (1 + 2 in a.b).__repr__() ** -3 ** 4""" - self.check(b, a) - - def test_5(self): - b = """x = a.has_key(f or g)""" - a = """x = (f or g) in a""" - self.check(b, a) - - def test_6(self): - b = """x = a + b.has_key(c)""" - a = """x = a + (c in b)""" - self.check(b, a) - - def test_7(self): - b = """x = a.has_key(lambda: 12)""" - a = """x = (lambda: 12) in a""" - self.check(b, a) - - def test_8(self): - b = """x = a.has_key(a for a in b)""" - a = """x = (a for a in b) in a""" - self.check(b, a) - - def test_9(self): - b = """if not a.has_key(b): pass""" - a = """if b not in a: pass""" - self.check(b, a) - - def test_10(self): - b = """if not a.has_key(b).__repr__(): pass""" - a = """if not (b in a).__repr__(): pass""" - self.check(b, a) - - def test_11(self): - b = """if not a.has_key(b) ** 2: pass""" - a = """if not (b in a) ** 2: pass""" - self.check(b, a) - -class Test_apply(FixerTestCase): - fixer = "apply" - - def test_1(self): - b = """x = apply(f, g + h)""" - a = """x = f(*g + h)""" - self.check(b, a) - - def test_2(self): - b = """y = apply(f, g, h)""" - a = """y = f(*g, **h)""" - self.check(b, a) - - def test_3(self): - b = """z = apply(fs[0], g or h, h or g)""" - a = """z = fs[0](*g or h, **h or g)""" - self.check(b, a) - - def test_4(self): - b = """apply(f, (x, y) + t)""" - a = """f(*(x, y) + t)""" - self.check(b, a) - - def test_5(self): - b = """apply(f, args,)""" - a = """f(*args)""" - self.check(b, a) - - def test_6(self): - b = """apply(f, args, kwds,)""" - a = """f(*args, **kwds)""" - self.check(b, a) - - # Test that complex functions are parenthesized - - def test_complex_1(self): - b = """x = apply(f+g, args)""" - a = """x = (f+g)(*args)""" - self.check(b, a) - - def test_complex_2(self): - b = """x = apply(f*g, args)""" - a = """x = (f*g)(*args)""" - self.check(b, a) - - def test_complex_3(self): - b = """x = apply(f**g, args)""" - a = """x = (f**g)(*args)""" - self.check(b, a) - - # But dotted names etc. not - - def test_dotted_name(self): - b = """x = apply(f.g, args)""" - a = """x = f.g(*args)""" - self.check(b, a) - - def test_subscript(self): - b = """x = apply(f[x], args)""" - a = """x = f[x](*args)""" - self.check(b, a) - - def test_call(self): - b = """x = apply(f(), args)""" - a = """x = f()(*args)""" - self.check(b, a) - - # Extreme case - def test_extreme(self): - b = """x = apply(a.b.c.d.e.f, args, kwds)""" - a = """x = a.b.c.d.e.f(*args, **kwds)""" - self.check(b, a) - - # XXX Comments in weird places still get lost - def test_weird_comments(self): - b = """apply( # foo - f, # bar - args)""" - a = """f(*args)""" - self.check(b, a) - - # These should *not* be touched - - def test_unchanged_1(self): - s = """apply()""" - self.unchanged(s) - - def test_unchanged_2(self): - s = """apply(f)""" - self.unchanged(s) - - def test_unchanged_3(self): - s = """apply(f,)""" - self.unchanged(s) - - def test_unchanged_4(self): - s = """apply(f, args, kwds, extras)""" - self.unchanged(s) - - def test_unchanged_5(self): - s = """apply(f, *args, **kwds)""" - self.unchanged(s) - - def test_unchanged_6(self): - s = """apply(f, *args)""" - self.unchanged(s) - - def test_unchanged_6b(self): - s = """apply(f, **kwds)""" - self.unchanged(s) - - def test_unchanged_7(self): - s = """apply(func=f, args=args, kwds=kwds)""" - self.unchanged(s) - - def test_unchanged_8(self): - s = """apply(f, args=args, kwds=kwds)""" - self.unchanged(s) - - def test_unchanged_9(self): - s = """apply(f, args, kwds=kwds)""" - self.unchanged(s) - - def test_space_1(self): - a = """apply( f, args, kwds)""" - b = """f(*args, **kwds)""" - self.check(a, b) - - def test_space_2(self): - a = """apply( f ,args,kwds )""" - b = """f(*args, **kwds)""" - self.check(a, b) - -class Test_reload(FixerTestCase): - fixer = "reload" - - def test(self): - b = """reload(a)""" - a = """import importlib\nimportlib.reload(a)""" - self.check(b, a) - - def test_comment(self): - b = """reload( a ) # comment""" - a = """import importlib\nimportlib.reload( a ) # comment""" - self.check(b, a) - - # PEP 8 comments - b = """reload( a ) # comment""" - a = """import importlib\nimportlib.reload( a ) # comment""" - self.check(b, a) - - def test_space(self): - b = """reload( a )""" - a = """import importlib\nimportlib.reload( a )""" - self.check(b, a) - - b = """reload( a)""" - a = """import importlib\nimportlib.reload( a)""" - self.check(b, a) - - b = """reload(a )""" - a = """import importlib\nimportlib.reload(a )""" - self.check(b, a) - - def test_unchanged(self): - s = """reload(a=1)""" - self.unchanged(s) - - s = """reload(f, g)""" - self.unchanged(s) - - s = """reload(f, *h)""" - self.unchanged(s) - - s = """reload(f, *h, **i)""" - self.unchanged(s) - - s = """reload(f, **i)""" - self.unchanged(s) - - s = """reload(*h, **i)""" - self.unchanged(s) - - s = """reload(*h)""" - self.unchanged(s) - - s = """reload(**i)""" - self.unchanged(s) - - s = """reload()""" - self.unchanged(s) - -class Test_intern(FixerTestCase): - fixer = "intern" - - def test_prefix_preservation(self): - b = """x = intern( a )""" - a = """import sys\nx = sys.intern( a )""" - self.check(b, a) - - b = """y = intern("b" # test - )""" - a = """import sys\ny = sys.intern("b" # test - )""" - self.check(b, a) - - b = """z = intern(a+b+c.d, )""" - a = """import sys\nz = sys.intern(a+b+c.d, )""" - self.check(b, a) - - def test(self): - b = """x = intern(a)""" - a = """import sys\nx = sys.intern(a)""" - self.check(b, a) - - b = """z = intern(a+b+c.d,)""" - a = """import sys\nz = sys.intern(a+b+c.d,)""" - self.check(b, a) - - b = """intern("y%s" % 5).replace("y", "")""" - a = """import sys\nsys.intern("y%s" % 5).replace("y", "")""" - self.check(b, a) - - # These should not be refactored - - def test_unchanged(self): - s = """intern(a=1)""" - self.unchanged(s) - - s = """intern(f, g)""" - self.unchanged(s) - - s = """intern(*h)""" - self.unchanged(s) - - s = """intern(**i)""" - self.unchanged(s) - - s = """intern()""" - self.unchanged(s) - -class Test_reduce(FixerTestCase): - fixer = "reduce" - - def test_simple_call(self): - b = "reduce(a, b, c)" - a = "from functools import reduce\nreduce(a, b, c)" - self.check(b, a) - - def test_bug_7253(self): - # fix_tuple_params was being bad and orphaning nodes in the tree. - b = "def x(arg): reduce(sum, [])" - a = "from functools import reduce\ndef x(arg): reduce(sum, [])" - self.check(b, a) - - def test_call_with_lambda(self): - b = "reduce(lambda x, y: x + y, seq)" - a = "from functools import reduce\nreduce(lambda x, y: x + y, seq)" - self.check(b, a) - - def test_unchanged(self): - s = "reduce(a)" - self.unchanged(s) - - s = "reduce(a, b=42)" - self.unchanged(s) - - s = "reduce(a, b, c, d)" - self.unchanged(s) - - s = "reduce(**c)" - self.unchanged(s) - - s = "reduce()" - self.unchanged(s) - -class Test_print(FixerTestCase): - fixer = "print" - - def test_prefix_preservation(self): - b = """print 1, 1+1, 1+1+1""" - a = """print(1, 1+1, 1+1+1)""" - self.check(b, a) - - def test_idempotency(self): - s = """print()""" - self.unchanged(s) - - s = """print('')""" - self.unchanged(s) - - def test_idempotency_print_as_function(self): - self.refactor.driver.grammar = pygram.python_grammar_no_print_statement - s = """print(1, 1+1, 1+1+1)""" - self.unchanged(s) - - s = """print()""" - self.unchanged(s) - - s = """print('')""" - self.unchanged(s) - - def test_1(self): - b = """print 1, 1+1, 1+1+1""" - a = """print(1, 1+1, 1+1+1)""" - self.check(b, a) - - def test_2(self): - b = """print 1, 2""" - a = """print(1, 2)""" - self.check(b, a) - - def test_3(self): - b = """print""" - a = """print()""" - self.check(b, a) - - def test_4(self): - # from bug 3000 - b = """print whatever; print""" - a = """print(whatever); print()""" - self.check(b, a) - - def test_5(self): - b = """print; print whatever;""" - a = """print(); print(whatever);""" - self.check(b, a) - - def test_tuple(self): - b = """print (a, b, c)""" - a = """print((a, b, c))""" - self.check(b, a) - - # trailing commas - - def test_trailing_comma_1(self): - b = """print 1, 2, 3,""" - a = """print(1, 2, 3, end=' ')""" - self.check(b, a) - - def test_trailing_comma_2(self): - b = """print 1, 2,""" - a = """print(1, 2, end=' ')""" - self.check(b, a) - - def test_trailing_comma_3(self): - b = """print 1,""" - a = """print(1, end=' ')""" - self.check(b, a) - - # >> stuff - - def test_vargs_without_trailing_comma(self): - b = """print >>sys.stderr, 1, 2, 3""" - a = """print(1, 2, 3, file=sys.stderr)""" - self.check(b, a) - - def test_with_trailing_comma(self): - b = """print >>sys.stderr, 1, 2,""" - a = """print(1, 2, end=' ', file=sys.stderr)""" - self.check(b, a) - - def test_no_trailing_comma(self): - b = """print >>sys.stderr, 1+1""" - a = """print(1+1, file=sys.stderr)""" - self.check(b, a) - - def test_spaces_before_file(self): - b = """print >> sys.stderr""" - a = """print(file=sys.stderr)""" - self.check(b, a) - - def test_with_future_print_function(self): - s = "from __future__ import print_function\n" \ - "print('Hai!', end=' ')" - self.unchanged(s) - - b = "print 'Hello, world!'" - a = "print('Hello, world!')" - self.check(b, a) - - -class Test_exec(FixerTestCase): - fixer = "exec" - - def test_prefix_preservation(self): - b = """ exec code in ns1, ns2""" - a = """ exec(code, ns1, ns2)""" - self.check(b, a) - - def test_basic(self): - b = """exec code""" - a = """exec(code)""" - self.check(b, a) - - def test_with_globals(self): - b = """exec code in ns""" - a = """exec(code, ns)""" - self.check(b, a) - - def test_with_globals_locals(self): - b = """exec code in ns1, ns2""" - a = """exec(code, ns1, ns2)""" - self.check(b, a) - - def test_complex_1(self): - b = """exec (a.b()) in ns""" - a = """exec((a.b()), ns)""" - self.check(b, a) - - def test_complex_2(self): - b = """exec a.b() + c in ns""" - a = """exec(a.b() + c, ns)""" - self.check(b, a) - - # These should not be touched - - def test_unchanged_1(self): - s = """exec(code)""" - self.unchanged(s) - - def test_unchanged_2(self): - s = """exec (code)""" - self.unchanged(s) - - def test_unchanged_3(self): - s = """exec(code, ns)""" - self.unchanged(s) - - def test_unchanged_4(self): - s = """exec(code, ns1, ns2)""" - self.unchanged(s) - -class Test_repr(FixerTestCase): - fixer = "repr" - - def test_prefix_preservation(self): - b = """x = `1 + 2`""" - a = """x = repr(1 + 2)""" - self.check(b, a) - - def test_simple_1(self): - b = """x = `1 + 2`""" - a = """x = repr(1 + 2)""" - self.check(b, a) - - def test_simple_2(self): - b = """y = `x`""" - a = """y = repr(x)""" - self.check(b, a) - - def test_complex(self): - b = """z = `y`.__repr__()""" - a = """z = repr(y).__repr__()""" - self.check(b, a) - - def test_tuple(self): - b = """x = `1, 2, 3`""" - a = """x = repr((1, 2, 3))""" - self.check(b, a) - - def test_nested(self): - b = """x = `1 + `2``""" - a = """x = repr(1 + repr(2))""" - self.check(b, a) - - def test_nested_tuples(self): - b = """x = `1, 2 + `3, 4``""" - a = """x = repr((1, 2 + repr((3, 4))))""" - self.check(b, a) - -class Test_except(FixerTestCase): - fixer = "except" - - def test_prefix_preservation(self): - b = """ - try: - pass - except (RuntimeError, ImportError), e: - pass""" - a = """ - try: - pass - except (RuntimeError, ImportError) as e: - pass""" - self.check(b, a) - - def test_simple(self): - b = """ - try: - pass - except Foo, e: - pass""" - a = """ - try: - pass - except Foo as e: - pass""" - self.check(b, a) - - def test_simple_no_space_before_target(self): - b = """ - try: - pass - except Foo,e: - pass""" - a = """ - try: - pass - except Foo as e: - pass""" - self.check(b, a) - - def test_tuple_unpack(self): - b = """ - def foo(): - try: - pass - except Exception, (f, e): - pass - except ImportError, e: - pass""" - - a = """ - def foo(): - try: - pass - except Exception as xxx_todo_changeme: - (f, e) = xxx_todo_changeme.args - pass - except ImportError as e: - pass""" - self.check(b, a) - - def test_multi_class(self): - b = """ - try: - pass - except (RuntimeError, ImportError), e: - pass""" - - a = """ - try: - pass - except (RuntimeError, ImportError) as e: - pass""" - self.check(b, a) - - def test_list_unpack(self): - b = """ - try: - pass - except Exception, [a, b]: - pass""" - - a = """ - try: - pass - except Exception as xxx_todo_changeme: - [a, b] = xxx_todo_changeme.args - pass""" - self.check(b, a) - - def test_weird_target_1(self): - b = """ - try: - pass - except Exception, d[5]: - pass""" - - a = """ - try: - pass - except Exception as xxx_todo_changeme: - d[5] = xxx_todo_changeme - pass""" - self.check(b, a) - - def test_weird_target_2(self): - b = """ - try: - pass - except Exception, a.foo: - pass""" - - a = """ - try: - pass - except Exception as xxx_todo_changeme: - a.foo = xxx_todo_changeme - pass""" - self.check(b, a) - - def test_weird_target_3(self): - b = """ - try: - pass - except Exception, a().foo: - pass""" - - a = """ - try: - pass - except Exception as xxx_todo_changeme: - a().foo = xxx_todo_changeme - pass""" - self.check(b, a) - - def test_bare_except(self): - b = """ - try: - pass - except Exception, a: - pass - except: - pass""" - - a = """ - try: - pass - except Exception as a: - pass - except: - pass""" - self.check(b, a) - - def test_bare_except_and_else_finally(self): - b = """ - try: - pass - except Exception, a: - pass - except: - pass - else: - pass - finally: - pass""" - - a = """ - try: - pass - except Exception as a: - pass - except: - pass - else: - pass - finally: - pass""" - self.check(b, a) - - def test_multi_fixed_excepts_before_bare_except(self): - b = """ - try: - pass - except TypeError, b: - pass - except Exception, a: - pass - except: - pass""" - - a = """ - try: - pass - except TypeError as b: - pass - except Exception as a: - pass - except: - pass""" - self.check(b, a) - - def test_one_line_suites(self): - b = """ - try: raise TypeError - except TypeError, e: - pass - """ - a = """ - try: raise TypeError - except TypeError as e: - pass - """ - self.check(b, a) - b = """ - try: - raise TypeError - except TypeError, e: pass - """ - a = """ - try: - raise TypeError - except TypeError as e: pass - """ - self.check(b, a) - b = """ - try: raise TypeError - except TypeError, e: pass - """ - a = """ - try: raise TypeError - except TypeError as e: pass - """ - self.check(b, a) - b = """ - try: raise TypeError - except TypeError, e: pass - else: function() - finally: done() - """ - a = """ - try: raise TypeError - except TypeError as e: pass - else: function() - finally: done() - """ - self.check(b, a) - - # These should not be touched: - - def test_unchanged_1(self): - s = """ - try: - pass - except: - pass""" - self.unchanged(s) - - def test_unchanged_2(self): - s = """ - try: - pass - except Exception: - pass""" - self.unchanged(s) - - def test_unchanged_3(self): - s = """ - try: - pass - except (Exception, SystemExit): - pass""" - self.unchanged(s) - -class Test_raise(FixerTestCase): - fixer = "raise" - - def test_basic(self): - b = """raise Exception, 5""" - a = """raise Exception(5)""" - self.check(b, a) - - def test_prefix_preservation(self): - b = """raise Exception,5""" - a = """raise Exception(5)""" - self.check(b, a) - - b = """raise Exception, 5""" - a = """raise Exception(5)""" - self.check(b, a) - - def test_with_comments(self): - b = """raise Exception, 5 # foo""" - a = """raise Exception(5) # foo""" - self.check(b, a) - - b = """raise E, (5, 6) % (a, b) # foo""" - a = """raise E((5, 6) % (a, b)) # foo""" - self.check(b, a) - - b = """def foo(): - raise Exception, 5, 6 # foo""" - a = """def foo(): - raise Exception(5).with_traceback(6) # foo""" - self.check(b, a) - - def test_None_value(self): - b = """raise Exception(5), None, tb""" - a = """raise Exception(5).with_traceback(tb)""" - self.check(b, a) - - def test_tuple_value(self): - b = """raise Exception, (5, 6, 7)""" - a = """raise Exception(5, 6, 7)""" - self.check(b, a) - - def test_tuple_detection(self): - b = """raise E, (5, 6) % (a, b)""" - a = """raise E((5, 6) % (a, b))""" - self.check(b, a) - - def test_tuple_exc_1(self): - b = """raise (((E1, E2), E3), E4), V""" - a = """raise E1(V)""" - self.check(b, a) - - def test_tuple_exc_2(self): - b = """raise (E1, (E2, E3), E4), V""" - a = """raise E1(V)""" - self.check(b, a) - - # These should produce a warning - - def test_string_exc(self): - s = """raise 'foo'""" - self.warns_unchanged(s, "Python 3 does not support string exceptions") - - def test_string_exc_val(self): - s = """raise "foo", 5""" - self.warns_unchanged(s, "Python 3 does not support string exceptions") - - def test_string_exc_val_tb(self): - s = """raise "foo", 5, 6""" - self.warns_unchanged(s, "Python 3 does not support string exceptions") - - # These should result in traceback-assignment - - def test_tb_1(self): - b = """def foo(): - raise Exception, 5, 6""" - a = """def foo(): - raise Exception(5).with_traceback(6)""" - self.check(b, a) - - def test_tb_2(self): - b = """def foo(): - a = 5 - raise Exception, 5, 6 - b = 6""" - a = """def foo(): - a = 5 - raise Exception(5).with_traceback(6) - b = 6""" - self.check(b, a) - - def test_tb_3(self): - b = """def foo(): - raise Exception,5,6""" - a = """def foo(): - raise Exception(5).with_traceback(6)""" - self.check(b, a) - - def test_tb_4(self): - b = """def foo(): - a = 5 - raise Exception,5,6 - b = 6""" - a = """def foo(): - a = 5 - raise Exception(5).with_traceback(6) - b = 6""" - self.check(b, a) - - def test_tb_5(self): - b = """def foo(): - raise Exception, (5, 6, 7), 6""" - a = """def foo(): - raise Exception(5, 6, 7).with_traceback(6)""" - self.check(b, a) - - def test_tb_6(self): - b = """def foo(): - a = 5 - raise Exception, (5, 6, 7), 6 - b = 6""" - a = """def foo(): - a = 5 - raise Exception(5, 6, 7).with_traceback(6) - b = 6""" - self.check(b, a) - -class Test_throw(FixerTestCase): - fixer = "throw" - - def test_1(self): - b = """g.throw(Exception, 5)""" - a = """g.throw(Exception(5))""" - self.check(b, a) - - def test_2(self): - b = """g.throw(Exception,5)""" - a = """g.throw(Exception(5))""" - self.check(b, a) - - def test_3(self): - b = """g.throw(Exception, (5, 6, 7))""" - a = """g.throw(Exception(5, 6, 7))""" - self.check(b, a) - - def test_4(self): - b = """5 + g.throw(Exception, 5)""" - a = """5 + g.throw(Exception(5))""" - self.check(b, a) - - # These should produce warnings - - def test_warn_1(self): - s = """g.throw("foo")""" - self.warns_unchanged(s, "Python 3 does not support string exceptions") - - def test_warn_2(self): - s = """g.throw("foo", 5)""" - self.warns_unchanged(s, "Python 3 does not support string exceptions") - - def test_warn_3(self): - s = """g.throw("foo", 5, 6)""" - self.warns_unchanged(s, "Python 3 does not support string exceptions") - - # These should not be touched - - def test_untouched_1(self): - s = """g.throw(Exception)""" - self.unchanged(s) - - def test_untouched_2(self): - s = """g.throw(Exception(5, 6))""" - self.unchanged(s) - - def test_untouched_3(self): - s = """5 + g.throw(Exception(5, 6))""" - self.unchanged(s) - - # These should result in traceback-assignment - - def test_tb_1(self): - b = """def foo(): - g.throw(Exception, 5, 6)""" - a = """def foo(): - g.throw(Exception(5).with_traceback(6))""" - self.check(b, a) - - def test_tb_2(self): - b = """def foo(): - a = 5 - g.throw(Exception, 5, 6) - b = 6""" - a = """def foo(): - a = 5 - g.throw(Exception(5).with_traceback(6)) - b = 6""" - self.check(b, a) - - def test_tb_3(self): - b = """def foo(): - g.throw(Exception,5,6)""" - a = """def foo(): - g.throw(Exception(5).with_traceback(6))""" - self.check(b, a) - - def test_tb_4(self): - b = """def foo(): - a = 5 - g.throw(Exception,5,6) - b = 6""" - a = """def foo(): - a = 5 - g.throw(Exception(5).with_traceback(6)) - b = 6""" - self.check(b, a) - - def test_tb_5(self): - b = """def foo(): - g.throw(Exception, (5, 6, 7), 6)""" - a = """def foo(): - g.throw(Exception(5, 6, 7).with_traceback(6))""" - self.check(b, a) - - def test_tb_6(self): - b = """def foo(): - a = 5 - g.throw(Exception, (5, 6, 7), 6) - b = 6""" - a = """def foo(): - a = 5 - g.throw(Exception(5, 6, 7).with_traceback(6)) - b = 6""" - self.check(b, a) - - def test_tb_7(self): - b = """def foo(): - a + g.throw(Exception, 5, 6)""" - a = """def foo(): - a + g.throw(Exception(5).with_traceback(6))""" - self.check(b, a) - - def test_tb_8(self): - b = """def foo(): - a = 5 - a + g.throw(Exception, 5, 6) - b = 6""" - a = """def foo(): - a = 5 - a + g.throw(Exception(5).with_traceback(6)) - b = 6""" - self.check(b, a) - -class Test_long(FixerTestCase): - fixer = "long" - - def test_1(self): - b = """x = long(x)""" - a = """x = int(x)""" - self.check(b, a) - - def test_2(self): - b = """y = isinstance(x, long)""" - a = """y = isinstance(x, int)""" - self.check(b, a) - - def test_3(self): - b = """z = type(x) in (int, long)""" - a = """z = type(x) in (int, int)""" - self.check(b, a) - - def test_unchanged(self): - s = """long = True""" - self.unchanged(s) - - s = """s.long = True""" - self.unchanged(s) - - s = """def long(): pass""" - self.unchanged(s) - - s = """class long(): pass""" - self.unchanged(s) - - s = """def f(long): pass""" - self.unchanged(s) - - s = """def f(g, long): pass""" - self.unchanged(s) - - s = """def f(x, long=True): pass""" - self.unchanged(s) - - def test_prefix_preservation(self): - b = """x = long( x )""" - a = """x = int( x )""" - self.check(b, a) - - -class Test_execfile(FixerTestCase): - fixer = "execfile" - - def test_conversion(self): - b = """execfile("fn")""" - a = """exec(compile(open("fn", "rb").read(), "fn", 'exec'))""" - self.check(b, a) - - b = """execfile("fn", glob)""" - a = """exec(compile(open("fn", "rb").read(), "fn", 'exec'), glob)""" - self.check(b, a) - - b = """execfile("fn", glob, loc)""" - a = """exec(compile(open("fn", "rb").read(), "fn", 'exec'), glob, loc)""" - self.check(b, a) - - b = """execfile("fn", globals=glob)""" - a = """exec(compile(open("fn", "rb").read(), "fn", 'exec'), globals=glob)""" - self.check(b, a) - - b = """execfile("fn", locals=loc)""" - a = """exec(compile(open("fn", "rb").read(), "fn", 'exec'), locals=loc)""" - self.check(b, a) - - b = """execfile("fn", globals=glob, locals=loc)""" - a = """exec(compile(open("fn", "rb").read(), "fn", 'exec'), globals=glob, locals=loc)""" - self.check(b, a) - - def test_spacing(self): - b = """execfile( "fn" )""" - a = """exec(compile(open( "fn", "rb" ).read(), "fn", 'exec'))""" - self.check(b, a) - - b = """execfile("fn", globals = glob)""" - a = """exec(compile(open("fn", "rb").read(), "fn", 'exec'), globals = glob)""" - self.check(b, a) - - -class Test_isinstance(FixerTestCase): - fixer = "isinstance" - - def test_remove_multiple_items(self): - b = """isinstance(x, (int, int, int))""" - a = """isinstance(x, int)""" - self.check(b, a) - - b = """isinstance(x, (int, float, int, int, float))""" - a = """isinstance(x, (int, float))""" - self.check(b, a) - - b = """isinstance(x, (int, float, int, int, float, str))""" - a = """isinstance(x, (int, float, str))""" - self.check(b, a) - - b = """isinstance(foo() + bar(), (x(), y(), x(), int, int))""" - a = """isinstance(foo() + bar(), (x(), y(), x(), int))""" - self.check(b, a) - - def test_prefix_preservation(self): - b = """if isinstance( foo(), ( bar, bar, baz )) : pass""" - a = """if isinstance( foo(), ( bar, baz )) : pass""" - self.check(b, a) - - def test_unchanged(self): - self.unchanged("isinstance(x, (str, int))") - -class Test_dict(FixerTestCase): - fixer = "dict" - - def test_prefix_preservation(self): - b = "if d. keys ( ) : pass" - a = "if list(d. keys ( )) : pass" - self.check(b, a) - - b = "if d. items ( ) : pass" - a = "if list(d. items ( )) : pass" - self.check(b, a) - - b = "if d. iterkeys ( ) : pass" - a = "if iter(d. keys ( )) : pass" - self.check(b, a) - - b = "[i for i in d. iterkeys( ) ]" - a = "[i for i in d. keys( ) ]" - self.check(b, a) - - b = "if d. viewkeys ( ) : pass" - a = "if d. keys ( ) : pass" - self.check(b, a) - - b = "[i for i in d. viewkeys( ) ]" - a = "[i for i in d. keys( ) ]" - self.check(b, a) - - def test_trailing_comment(self): - b = "d.keys() # foo" - a = "list(d.keys()) # foo" - self.check(b, a) - - b = "d.items() # foo" - a = "list(d.items()) # foo" - self.check(b, a) - - b = "d.iterkeys() # foo" - a = "iter(d.keys()) # foo" - self.check(b, a) - - b = """[i for i in d.iterkeys() # foo - ]""" - a = """[i for i in d.keys() # foo - ]""" - self.check(b, a) - - b = """[i for i in d.iterkeys() # foo - ]""" - a = """[i for i in d.keys() # foo - ]""" - self.check(b, a) - - b = "d.viewitems() # foo" - a = "d.items() # foo" - self.check(b, a) - - def test_unchanged(self): - for wrapper in fixer_util.consuming_calls: - s = "s = %s(d.keys())" % wrapper - self.unchanged(s) - - s = "s = %s(d.values())" % wrapper - self.unchanged(s) - - s = "s = %s(d.items())" % wrapper - self.unchanged(s) - - def test_01(self): - b = "d.keys()" - a = "list(d.keys())" - self.check(b, a) - - b = "a[0].foo().keys()" - a = "list(a[0].foo().keys())" - self.check(b, a) - - def test_02(self): - b = "d.items()" - a = "list(d.items())" - self.check(b, a) - - def test_03(self): - b = "d.values()" - a = "list(d.values())" - self.check(b, a) - - def test_04(self): - b = "d.iterkeys()" - a = "iter(d.keys())" - self.check(b, a) - - def test_05(self): - b = "d.iteritems()" - a = "iter(d.items())" - self.check(b, a) - - def test_06(self): - b = "d.itervalues()" - a = "iter(d.values())" - self.check(b, a) - - def test_07(self): - s = "list(d.keys())" - self.unchanged(s) - - def test_08(self): - s = "sorted(d.keys())" - self.unchanged(s) - - def test_09(self): - b = "iter(d.keys())" - a = "iter(list(d.keys()))" - self.check(b, a) - - def test_10(self): - b = "foo(d.keys())" - a = "foo(list(d.keys()))" - self.check(b, a) - - def test_11(self): - b = "for i in d.keys(): print i" - a = "for i in list(d.keys()): print i" - self.check(b, a) - - def test_12(self): - b = "for i in d.iterkeys(): print i" - a = "for i in d.keys(): print i" - self.check(b, a) - - def test_13(self): - b = "[i for i in d.keys()]" - a = "[i for i in list(d.keys())]" - self.check(b, a) - - def test_14(self): - b = "[i for i in d.iterkeys()]" - a = "[i for i in d.keys()]" - self.check(b, a) - - def test_15(self): - b = "(i for i in d.keys())" - a = "(i for i in list(d.keys()))" - self.check(b, a) - - def test_16(self): - b = "(i for i in d.iterkeys())" - a = "(i for i in d.keys())" - self.check(b, a) - - def test_17(self): - b = "iter(d.iterkeys())" - a = "iter(d.keys())" - self.check(b, a) - - def test_18(self): - b = "list(d.iterkeys())" - a = "list(d.keys())" - self.check(b, a) - - def test_19(self): - b = "sorted(d.iterkeys())" - a = "sorted(d.keys())" - self.check(b, a) - - def test_20(self): - b = "foo(d.iterkeys())" - a = "foo(iter(d.keys()))" - self.check(b, a) - - def test_21(self): - b = "print h.iterkeys().next()" - a = "print iter(h.keys()).next()" - self.check(b, a) - - def test_22(self): - b = "print h.keys()[0]" - a = "print list(h.keys())[0]" - self.check(b, a) - - def test_23(self): - b = "print list(h.iterkeys().next())" - a = "print list(iter(h.keys()).next())" - self.check(b, a) - - def test_24(self): - b = "for x in h.keys()[0]: print x" - a = "for x in list(h.keys())[0]: print x" - self.check(b, a) - - def test_25(self): - b = "d.viewkeys()" - a = "d.keys()" - self.check(b, a) - - def test_26(self): - b = "d.viewitems()" - a = "d.items()" - self.check(b, a) - - def test_27(self): - b = "d.viewvalues()" - a = "d.values()" - self.check(b, a) - - def test_28(self): - b = "[i for i in d.viewkeys()]" - a = "[i for i in d.keys()]" - self.check(b, a) - - def test_29(self): - b = "(i for i in d.viewkeys())" - a = "(i for i in d.keys())" - self.check(b, a) - - def test_30(self): - b = "iter(d.viewkeys())" - a = "iter(d.keys())" - self.check(b, a) - - def test_31(self): - b = "list(d.viewkeys())" - a = "list(d.keys())" - self.check(b, a) - - def test_32(self): - b = "sorted(d.viewkeys())" - a = "sorted(d.keys())" - self.check(b, a) - -class Test_xrange(FixerTestCase): - fixer = "xrange" - - def test_prefix_preservation(self): - b = """x = xrange( 10 )""" - a = """x = range( 10 )""" - self.check(b, a) - - b = """x = xrange( 1 , 10 )""" - a = """x = range( 1 , 10 )""" - self.check(b, a) - - b = """x = xrange( 0 , 10 , 2 )""" - a = """x = range( 0 , 10 , 2 )""" - self.check(b, a) - - def test_single_arg(self): - b = """x = xrange(10)""" - a = """x = range(10)""" - self.check(b, a) - - def test_two_args(self): - b = """x = xrange(1, 10)""" - a = """x = range(1, 10)""" - self.check(b, a) - - def test_three_args(self): - b = """x = xrange(0, 10, 2)""" - a = """x = range(0, 10, 2)""" - self.check(b, a) - - def test_wrap_in_list(self): - b = """x = range(10, 3, 9)""" - a = """x = list(range(10, 3, 9))""" - self.check(b, a) - - b = """x = foo(range(10, 3, 9))""" - a = """x = foo(list(range(10, 3, 9)))""" - self.check(b, a) - - b = """x = range(10, 3, 9) + [4]""" - a = """x = list(range(10, 3, 9)) + [4]""" - self.check(b, a) - - b = """x = range(10)[::-1]""" - a = """x = list(range(10))[::-1]""" - self.check(b, a) - - b = """x = range(10) [3]""" - a = """x = list(range(10)) [3]""" - self.check(b, a) - - def test_xrange_in_for(self): - b = """for i in xrange(10):\n j=i""" - a = """for i in range(10):\n j=i""" - self.check(b, a) - - b = """[i for i in xrange(10)]""" - a = """[i for i in range(10)]""" - self.check(b, a) - - def test_range_in_for(self): - self.unchanged("for i in range(10): pass") - self.unchanged("[i for i in range(10)]") - - def test_in_contains_test(self): - self.unchanged("x in range(10, 3, 9)") - - def test_in_consuming_context(self): - for call in fixer_util.consuming_calls: - self.unchanged("a = %s(range(10))" % call) - -class Test_xrange_with_reduce(FixerTestCase): - - def setUp(self): - super(Test_xrange_with_reduce, self).setUp(["xrange", "reduce"]) - - def test_double_transform(self): - b = """reduce(x, xrange(5))""" - a = """from functools import reduce -reduce(x, range(5))""" - self.check(b, a) - -class Test_raw_input(FixerTestCase): - fixer = "raw_input" - - def test_prefix_preservation(self): - b = """x = raw_input( )""" - a = """x = input( )""" - self.check(b, a) - - b = """x = raw_input( '' )""" - a = """x = input( '' )""" - self.check(b, a) - - def test_1(self): - b = """x = raw_input()""" - a = """x = input()""" - self.check(b, a) - - def test_2(self): - b = """x = raw_input('')""" - a = """x = input('')""" - self.check(b, a) - - def test_3(self): - b = """x = raw_input('prompt')""" - a = """x = input('prompt')""" - self.check(b, a) - - def test_4(self): - b = """x = raw_input(foo(a) + 6)""" - a = """x = input(foo(a) + 6)""" - self.check(b, a) - - def test_5(self): - b = """x = raw_input(invite).split()""" - a = """x = input(invite).split()""" - self.check(b, a) - - def test_6(self): - b = """x = raw_input(invite) . split ()""" - a = """x = input(invite) . split ()""" - self.check(b, a) - - def test_8(self): - b = "x = int(raw_input())" - a = "x = int(input())" - self.check(b, a) - -class Test_funcattrs(FixerTestCase): - fixer = "funcattrs" - - attrs = ["closure", "doc", "name", "defaults", "code", "globals", "dict"] - - def test(self): - for attr in self.attrs: - b = "a.func_%s" % attr - a = "a.__%s__" % attr - self.check(b, a) - - b = "self.foo.func_%s.foo_bar" % attr - a = "self.foo.__%s__.foo_bar" % attr - self.check(b, a) - - def test_unchanged(self): - for attr in self.attrs: - s = "foo(func_%s + 5)" % attr - self.unchanged(s) - - s = "f(foo.__%s__)" % attr - self.unchanged(s) - - s = "f(foo.__%s__.foo)" % attr - self.unchanged(s) - -class Test_xreadlines(FixerTestCase): - fixer = "xreadlines" - - def test_call(self): - b = "for x in f.xreadlines(): pass" - a = "for x in f: pass" - self.check(b, a) - - b = "for x in foo().xreadlines(): pass" - a = "for x in foo(): pass" - self.check(b, a) - - b = "for x in (5 + foo()).xreadlines(): pass" - a = "for x in (5 + foo()): pass" - self.check(b, a) - - def test_attr_ref(self): - b = "foo(f.xreadlines + 5)" - a = "foo(f.__iter__ + 5)" - self.check(b, a) - - b = "foo(f().xreadlines + 5)" - a = "foo(f().__iter__ + 5)" - self.check(b, a) - - b = "foo((5 + f()).xreadlines + 5)" - a = "foo((5 + f()).__iter__ + 5)" - self.check(b, a) - - def test_unchanged(self): - s = "for x in f.xreadlines(5): pass" - self.unchanged(s) - - s = "for x in f.xreadlines(k=5): pass" - self.unchanged(s) - - s = "for x in f.xreadlines(*k, **v): pass" - self.unchanged(s) - - s = "foo(xreadlines)" - self.unchanged(s) - - -class ImportsFixerTests: - - def test_import_module(self): - for old, new in self.modules.items(): - b = "import %s" % old - a = "import %s" % new - self.check(b, a) - - b = "import foo, %s, bar" % old - a = "import foo, %s, bar" % new - self.check(b, a) - - def test_import_from(self): - for old, new in self.modules.items(): - b = "from %s import foo" % old - a = "from %s import foo" % new - self.check(b, a) - - b = "from %s import foo, bar" % old - a = "from %s import foo, bar" % new - self.check(b, a) - - b = "from %s import (yes, no)" % old - a = "from %s import (yes, no)" % new - self.check(b, a) - - def test_import_module_as(self): - for old, new in self.modules.items(): - b = "import %s as foo_bar" % old - a = "import %s as foo_bar" % new - self.check(b, a) - - b = "import %s as foo_bar" % old - a = "import %s as foo_bar" % new - self.check(b, a) - - def test_import_from_as(self): - for old, new in self.modules.items(): - b = "from %s import foo as bar" % old - a = "from %s import foo as bar" % new - self.check(b, a) - - def test_star(self): - for old, new in self.modules.items(): - b = "from %s import *" % old - a = "from %s import *" % new - self.check(b, a) - - def test_import_module_usage(self): - for old, new in self.modules.items(): - b = """ - import %s - foo(%s.bar) - """ % (old, old) - a = """ - import %s - foo(%s.bar) - """ % (new, new) - self.check(b, a) - - b = """ - from %s import x - %s = 23 - """ % (old, old) - a = """ - from %s import x - %s = 23 - """ % (new, old) - self.check(b, a) - - s = """ - def f(): - %s.method() - """ % (old,) - self.unchanged(s) - - # test nested usage - b = """ - import %s - %s.bar(%s.foo) - """ % (old, old, old) - a = """ - import %s - %s.bar(%s.foo) - """ % (new, new, new) - self.check(b, a) - - b = """ - import %s - x.%s - """ % (old, old) - a = """ - import %s - x.%s - """ % (new, old) - self.check(b, a) - - -class Test_imports(FixerTestCase, ImportsFixerTests): - fixer = "imports" - from lib2to3.fixes.fix_imports import MAPPING as modules - - def test_multiple_imports(self): - b = """import urlparse, cStringIO""" - a = """import urllib.parse, io""" - self.check(b, a) - - def test_multiple_imports_as(self): - b = """ - import copy_reg as bar, HTMLParser as foo, urlparse - s = urlparse.spam(bar.foo()) - """ - a = """ - import copyreg as bar, html.parser as foo, urllib.parse - s = urllib.parse.spam(bar.foo()) - """ - self.check(b, a) - - -class Test_imports2(FixerTestCase, ImportsFixerTests): - fixer = "imports2" - from lib2to3.fixes.fix_imports2 import MAPPING as modules - - -class Test_imports_fixer_order(FixerTestCase, ImportsFixerTests): - - def setUp(self): - super(Test_imports_fixer_order, self).setUp(['imports', 'imports2']) - from lib2to3.fixes.fix_imports2 import MAPPING as mapping2 - self.modules = mapping2.copy() - from lib2to3.fixes.fix_imports import MAPPING as mapping1 - for key in ('dbhash', 'dumbdbm', 'dbm', 'gdbm'): - self.modules[key] = mapping1[key] - - def test_after_local_imports_refactoring(self): - for fix in ("imports", "imports2"): - self.fixer = fix - self.assert_runs_after("import") - - -class Test_urllib(FixerTestCase): - fixer = "urllib" - from lib2to3.fixes.fix_urllib import MAPPING as modules - - def test_import_module(self): - for old, changes in self.modules.items(): - b = "import %s" % old - a = "import %s" % ", ".join(map(itemgetter(0), changes)) - self.check(b, a) - - def test_import_from(self): - for old, changes in self.modules.items(): - all_members = [] - for new, members in changes: - for member in members: - all_members.append(member) - b = "from %s import %s" % (old, member) - a = "from %s import %s" % (new, member) - self.check(b, a) - - s = "from foo import %s" % member - self.unchanged(s) - - b = "from %s import %s" % (old, ", ".join(members)) - a = "from %s import %s" % (new, ", ".join(members)) - self.check(b, a) - - s = "from foo import %s" % ", ".join(members) - self.unchanged(s) - - # test the breaking of a module into multiple replacements - b = "from %s import %s" % (old, ", ".join(all_members)) - a = "\n".join(["from %s import %s" % (new, ", ".join(members)) - for (new, members) in changes]) - self.check(b, a) - - def test_import_module_as(self): - for old in self.modules: - s = "import %s as foo" % old - self.warns_unchanged(s, "This module is now multiple modules") - - def test_import_from_as(self): - for old, changes in self.modules.items(): - for new, members in changes: - for member in members: - b = "from %s import %s as foo_bar" % (old, member) - a = "from %s import %s as foo_bar" % (new, member) - self.check(b, a) - b = "from %s import %s as blah, %s" % (old, member, member) - a = "from %s import %s as blah, %s" % (new, member, member) - self.check(b, a) - - def test_star(self): - for old in self.modules: - s = "from %s import *" % old - self.warns_unchanged(s, "Cannot handle star imports") - - def test_indented(self): - b = """ -def foo(): - from urllib import urlencode, urlopen -""" - a = """ -def foo(): - from urllib.parse import urlencode - from urllib.request import urlopen -""" - self.check(b, a) - - b = """ -def foo(): - other() - from urllib import urlencode, urlopen -""" - a = """ -def foo(): - other() - from urllib.parse import urlencode - from urllib.request import urlopen -""" - self.check(b, a) - - def test_single_import(self): - b = "from urllib import getproxies" - a = "from urllib.request import getproxies" - - self.check(b, a) - - def test_import_module_usage(self): - for old, changes in self.modules.items(): - for new, members in changes: - for member in members: - new_import = ", ".join([n for (n, mems) - in self.modules[old]]) - b = """ - import %s - foo(%s.%s) - """ % (old, old, member) - a = """ - import %s - foo(%s.%s) - """ % (new_import, new, member) - self.check(b, a) - b = """ - import %s - %s.%s(%s.%s) - """ % (old, old, member, old, member) - a = """ - import %s - %s.%s(%s.%s) - """ % (new_import, new, member, new, member) - self.check(b, a) - - -class Test_input(FixerTestCase): - fixer = "input" - - def test_prefix_preservation(self): - b = """x = input( )""" - a = """x = eval(input( ))""" - self.check(b, a) - - b = """x = input( '' )""" - a = """x = eval(input( '' ))""" - self.check(b, a) - - def test_trailing_comment(self): - b = """x = input() # foo""" - a = """x = eval(input()) # foo""" - self.check(b, a) - - def test_idempotency(self): - s = """x = eval(input())""" - self.unchanged(s) - - s = """x = eval(input(''))""" - self.unchanged(s) - - s = """x = eval(input(foo(5) + 9))""" - self.unchanged(s) - - def test_1(self): - b = """x = input()""" - a = """x = eval(input())""" - self.check(b, a) - - def test_2(self): - b = """x = input('')""" - a = """x = eval(input(''))""" - self.check(b, a) - - def test_3(self): - b = """x = input('prompt')""" - a = """x = eval(input('prompt'))""" - self.check(b, a) - - def test_4(self): - b = """x = input(foo(5) + 9)""" - a = """x = eval(input(foo(5) + 9))""" - self.check(b, a) - -class Test_tuple_params(FixerTestCase): - fixer = "tuple_params" - - def test_unchanged_1(self): - s = """def foo(): pass""" - self.unchanged(s) - - def test_unchanged_2(self): - s = """def foo(a, b, c): pass""" - self.unchanged(s) - - def test_unchanged_3(self): - s = """def foo(a=3, b=4, c=5): pass""" - self.unchanged(s) - - def test_1(self): - b = """ - def foo(((a, b), c)): - x = 5""" - - a = """ - def foo(xxx_todo_changeme): - ((a, b), c) = xxx_todo_changeme - x = 5""" - self.check(b, a) - - def test_2(self): - b = """ - def foo(((a, b), c), d): - x = 5""" - - a = """ - def foo(xxx_todo_changeme, d): - ((a, b), c) = xxx_todo_changeme - x = 5""" - self.check(b, a) - - def test_3(self): - b = """ - def foo(((a, b), c), d) -> e: - x = 5""" - - a = """ - def foo(xxx_todo_changeme, d) -> e: - ((a, b), c) = xxx_todo_changeme - x = 5""" - self.check(b, a) - - def test_semicolon(self): - b = """ - def foo(((a, b), c)): x = 5; y = 7""" - - a = """ - def foo(xxx_todo_changeme): ((a, b), c) = xxx_todo_changeme; x = 5; y = 7""" - self.check(b, a) - - def test_keywords(self): - b = """ - def foo(((a, b), c), d, e=5) -> z: - x = 5""" - - a = """ - def foo(xxx_todo_changeme, d, e=5) -> z: - ((a, b), c) = xxx_todo_changeme - x = 5""" - self.check(b, a) - - def test_varargs(self): - b = """ - def foo(((a, b), c), d, *vargs, **kwargs) -> z: - x = 5""" - - a = """ - def foo(xxx_todo_changeme, d, *vargs, **kwargs) -> z: - ((a, b), c) = xxx_todo_changeme - x = 5""" - self.check(b, a) - - def test_multi_1(self): - b = """ - def foo(((a, b), c), (d, e, f)) -> z: - x = 5""" - - a = """ - def foo(xxx_todo_changeme, xxx_todo_changeme1) -> z: - ((a, b), c) = xxx_todo_changeme - (d, e, f) = xxx_todo_changeme1 - x = 5""" - self.check(b, a) - - def test_multi_2(self): - b = """ - def foo(x, ((a, b), c), d, (e, f, g), y) -> z: - x = 5""" - - a = """ - def foo(x, xxx_todo_changeme, d, xxx_todo_changeme1, y) -> z: - ((a, b), c) = xxx_todo_changeme - (e, f, g) = xxx_todo_changeme1 - x = 5""" - self.check(b, a) - - def test_docstring(self): - b = """ - def foo(((a, b), c), (d, e, f)) -> z: - "foo foo foo foo" - x = 5""" - - a = """ - def foo(xxx_todo_changeme, xxx_todo_changeme1) -> z: - "foo foo foo foo" - ((a, b), c) = xxx_todo_changeme - (d, e, f) = xxx_todo_changeme1 - x = 5""" - self.check(b, a) - - def test_lambda_no_change(self): - s = """lambda x: x + 5""" - self.unchanged(s) - - def test_lambda_parens_single_arg(self): - b = """lambda (x): x + 5""" - a = """lambda x: x + 5""" - self.check(b, a) - - b = """lambda(x): x + 5""" - a = """lambda x: x + 5""" - self.check(b, a) - - b = """lambda ((((x)))): x + 5""" - a = """lambda x: x + 5""" - self.check(b, a) - - b = """lambda((((x)))): x + 5""" - a = """lambda x: x + 5""" - self.check(b, a) - - def test_lambda_simple(self): - b = """lambda (x, y): x + f(y)""" - a = """lambda x_y: x_y[0] + f(x_y[1])""" - self.check(b, a) - - b = """lambda(x, y): x + f(y)""" - a = """lambda x_y: x_y[0] + f(x_y[1])""" - self.check(b, a) - - b = """lambda (((x, y))): x + f(y)""" - a = """lambda x_y: x_y[0] + f(x_y[1])""" - self.check(b, a) - - b = """lambda(((x, y))): x + f(y)""" - a = """lambda x_y: x_y[0] + f(x_y[1])""" - self.check(b, a) - - def test_lambda_one_tuple(self): - b = """lambda (x,): x + f(x)""" - a = """lambda x1: x1[0] + f(x1[0])""" - self.check(b, a) - - b = """lambda (((x,))): x + f(x)""" - a = """lambda x1: x1[0] + f(x1[0])""" - self.check(b, a) - - def test_lambda_simple_multi_use(self): - b = """lambda (x, y): x + x + f(x) + x""" - a = """lambda x_y: x_y[0] + x_y[0] + f(x_y[0]) + x_y[0]""" - self.check(b, a) - - def test_lambda_simple_reverse(self): - b = """lambda (x, y): y + x""" - a = """lambda x_y: x_y[1] + x_y[0]""" - self.check(b, a) - - def test_lambda_nested(self): - b = """lambda (x, (y, z)): x + y + z""" - a = """lambda x_y_z: x_y_z[0] + x_y_z[1][0] + x_y_z[1][1]""" - self.check(b, a) - - b = """lambda (((x, (y, z)))): x + y + z""" - a = """lambda x_y_z: x_y_z[0] + x_y_z[1][0] + x_y_z[1][1]""" - self.check(b, a) - - def test_lambda_nested_multi_use(self): - b = """lambda (x, (y, z)): x + y + f(y)""" - a = """lambda x_y_z: x_y_z[0] + x_y_z[1][0] + f(x_y_z[1][0])""" - self.check(b, a) - -class Test_methodattrs(FixerTestCase): - fixer = "methodattrs" - - attrs = ["func", "self", "class"] - - def test(self): - for attr in self.attrs: - b = "a.im_%s" % attr - if attr == "class": - a = "a.__self__.__class__" - else: - a = "a.__%s__" % attr - self.check(b, a) - - b = "self.foo.im_%s.foo_bar" % attr - if attr == "class": - a = "self.foo.__self__.__class__.foo_bar" - else: - a = "self.foo.__%s__.foo_bar" % attr - self.check(b, a) - - def test_unchanged(self): - for attr in self.attrs: - s = "foo(im_%s + 5)" % attr - self.unchanged(s) - - s = "f(foo.__%s__)" % attr - self.unchanged(s) - - s = "f(foo.__%s__.foo)" % attr - self.unchanged(s) - -class Test_next(FixerTestCase): - fixer = "next" - - def test_1(self): - b = """it.next()""" - a = """next(it)""" - self.check(b, a) - - def test_2(self): - b = """a.b.c.d.next()""" - a = """next(a.b.c.d)""" - self.check(b, a) - - def test_3(self): - b = """(a + b).next()""" - a = """next((a + b))""" - self.check(b, a) - - def test_4(self): - b = """a().next()""" - a = """next(a())""" - self.check(b, a) - - def test_5(self): - b = """a().next() + b""" - a = """next(a()) + b""" - self.check(b, a) - - def test_6(self): - b = """c( a().next() + b)""" - a = """c( next(a()) + b)""" - self.check(b, a) - - def test_prefix_preservation_1(self): - b = """ - for a in b: - foo(a) - a.next() - """ - a = """ - for a in b: - foo(a) - next(a) - """ - self.check(b, a) - - def test_prefix_preservation_2(self): - b = """ - for a in b: - foo(a) # abc - # def - a.next() - """ - a = """ - for a in b: - foo(a) # abc - # def - next(a) - """ - self.check(b, a) - - def test_prefix_preservation_3(self): - b = """ - next = 5 - for a in b: - foo(a) - a.next() - """ - a = """ - next = 5 - for a in b: - foo(a) - a.__next__() - """ - self.check(b, a, ignore_warnings=True) - - def test_prefix_preservation_4(self): - b = """ - next = 5 - for a in b: - foo(a) # abc - # def - a.next() - """ - a = """ - next = 5 - for a in b: - foo(a) # abc - # def - a.__next__() - """ - self.check(b, a, ignore_warnings=True) - - def test_prefix_preservation_5(self): - b = """ - next = 5 - for a in b: - foo(foo(a), # abc - a.next()) - """ - a = """ - next = 5 - for a in b: - foo(foo(a), # abc - a.__next__()) - """ - self.check(b, a, ignore_warnings=True) - - def test_prefix_preservation_6(self): - b = """ - for a in b: - foo(foo(a), # abc - a.next()) - """ - a = """ - for a in b: - foo(foo(a), # abc - next(a)) - """ - self.check(b, a) - - def test_method_1(self): - b = """ - class A: - def next(self): - pass - """ - a = """ - class A: - def __next__(self): - pass - """ - self.check(b, a) - - def test_method_2(self): - b = """ - class A(object): - def next(self): - pass - """ - a = """ - class A(object): - def __next__(self): - pass - """ - self.check(b, a) - - def test_method_3(self): - b = """ - class A: - def next(x): - pass - """ - a = """ - class A: - def __next__(x): - pass - """ - self.check(b, a) - - def test_method_4(self): - b = """ - class A: - def __init__(self, foo): - self.foo = foo - - def next(self): - pass - - def __iter__(self): - return self - """ - a = """ - class A: - def __init__(self, foo): - self.foo = foo - - def __next__(self): - pass - - def __iter__(self): - return self - """ - self.check(b, a) - - def test_method_unchanged(self): - s = """ - class A: - def next(self, a, b): - pass - """ - self.unchanged(s) - - def test_shadowing_assign_simple(self): - s = """ - next = foo - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_assign_tuple_1(self): - s = """ - (next, a) = foo - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_assign_tuple_2(self): - s = """ - (a, (b, (next, c)), a) = foo - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_assign_list_1(self): - s = """ - [next, a] = foo - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_assign_list_2(self): - s = """ - [a, [b, [next, c]], a] = foo - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_builtin_assign(self): - s = """ - def foo(): - __builtin__.next = foo - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_builtin_assign_in_tuple(self): - s = """ - def foo(): - (a, __builtin__.next) = foo - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_builtin_assign_in_list(self): - s = """ - def foo(): - [a, __builtin__.next] = foo - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_assign_to_next(self): - s = """ - def foo(): - A.next = foo - - class A: - def next(self, a, b): - pass - """ - self.unchanged(s) - - def test_assign_to_next_in_tuple(self): - s = """ - def foo(): - (a, A.next) = foo - - class A: - def next(self, a, b): - pass - """ - self.unchanged(s) - - def test_assign_to_next_in_list(self): - s = """ - def foo(): - [a, A.next] = foo - - class A: - def next(self, a, b): - pass - """ - self.unchanged(s) - - def test_shadowing_import_1(self): - s = """ - import foo.bar as next - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_import_2(self): - s = """ - import bar, bar.foo as next - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_import_3(self): - s = """ - import bar, bar.foo as next, baz - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_import_from_1(self): - s = """ - from x import next - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_import_from_2(self): - s = """ - from x.a import next - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_import_from_3(self): - s = """ - from x import a, next, b - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_import_from_4(self): - s = """ - from x.a import a, next, b - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_funcdef_1(self): - s = """ - def next(a): - pass - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_funcdef_2(self): - b = """ - def next(a): - pass - - class A: - def next(self): - pass - - it.next() - """ - a = """ - def next(a): - pass - - class A: - def __next__(self): - pass - - it.__next__() - """ - self.warns(b, a, "Calls to builtin next() possibly shadowed") - - def test_shadowing_global_1(self): - s = """ - def f(): - global next - next = 5 - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_global_2(self): - s = """ - def f(): - global a, next, b - next = 5 - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_for_simple(self): - s = """ - for next in it(): - pass - - b = 5 - c = 6 - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_for_tuple_1(self): - s = """ - for next, b in it(): - pass - - b = 5 - c = 6 - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_for_tuple_2(self): - s = """ - for a, (next, c), b in it(): - pass - - b = 5 - c = 6 - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_noncall_access_1(self): - b = """gnext = g.next""" - a = """gnext = g.__next__""" - self.check(b, a) - - def test_noncall_access_2(self): - b = """f(g.next + 5)""" - a = """f(g.__next__ + 5)""" - self.check(b, a) - - def test_noncall_access_3(self): - b = """f(g().next + 5)""" - a = """f(g().__next__ + 5)""" - self.check(b, a) - -class Test_nonzero(FixerTestCase): - fixer = "nonzero" - - def test_1(self): - b = """ - class A: - def __nonzero__(self): - pass - """ - a = """ - class A: - def __bool__(self): - pass - """ - self.check(b, a) - - def test_2(self): - b = """ - class A(object): - def __nonzero__(self): - pass - """ - a = """ - class A(object): - def __bool__(self): - pass - """ - self.check(b, a) - - def test_unchanged_1(self): - s = """ - class A(object): - def __bool__(self): - pass - """ - self.unchanged(s) - - def test_unchanged_2(self): - s = """ - class A(object): - def __nonzero__(self, a): - pass - """ - self.unchanged(s) - - def test_unchanged_func(self): - s = """ - def __nonzero__(self): - pass - """ - self.unchanged(s) - -class Test_numliterals(FixerTestCase): - fixer = "numliterals" - - def test_octal_1(self): - b = """0755""" - a = """0o755""" - self.check(b, a) - - def test_long_int_1(self): - b = """a = 12L""" - a = """a = 12""" - self.check(b, a) - - def test_long_int_2(self): - b = """a = 12l""" - a = """a = 12""" - self.check(b, a) - - def test_long_hex(self): - b = """b = 0x12l""" - a = """b = 0x12""" - self.check(b, a) - - def test_comments_and_spacing(self): - b = """b = 0x12L""" - a = """b = 0x12""" - self.check(b, a) - - b = """b = 0755 # spam""" - a = """b = 0o755 # spam""" - self.check(b, a) - - def test_unchanged_int(self): - s = """5""" - self.unchanged(s) - - def test_unchanged_float(self): - s = """5.0""" - self.unchanged(s) - - def test_unchanged_octal(self): - s = """0o755""" - self.unchanged(s) - - def test_unchanged_hex(self): - s = """0xABC""" - self.unchanged(s) - - def test_unchanged_exp(self): - s = """5.0e10""" - self.unchanged(s) - - def test_unchanged_complex_int(self): - s = """5 + 4j""" - self.unchanged(s) - - def test_unchanged_complex_float(self): - s = """5.4 + 4.9j""" - self.unchanged(s) - - def test_unchanged_complex_bare(self): - s = """4j""" - self.unchanged(s) - s = """4.4j""" - self.unchanged(s) - -class Test_renames(FixerTestCase): - fixer = "renames" - - modules = {"sys": ("maxint", "maxsize"), - } - - def test_import_from(self): - for mod, (old, new) in list(self.modules.items()): - b = "from %s import %s" % (mod, old) - a = "from %s import %s" % (mod, new) - self.check(b, a) - - s = "from foo import %s" % old - self.unchanged(s) - - def test_import_from_as(self): - for mod, (old, new) in list(self.modules.items()): - b = "from %s import %s as foo_bar" % (mod, old) - a = "from %s import %s as foo_bar" % (mod, new) - self.check(b, a) - - def test_import_module_usage(self): - for mod, (old, new) in list(self.modules.items()): - b = """ - import %s - foo(%s, %s.%s) - """ % (mod, mod, mod, old) - a = """ - import %s - foo(%s, %s.%s) - """ % (mod, mod, mod, new) - self.check(b, a) - - def XXX_test_from_import_usage(self): - # not implemented yet - for mod, (old, new) in list(self.modules.items()): - b = """ - from %s import %s - foo(%s, %s) - """ % (mod, old, mod, old) - a = """ - from %s import %s - foo(%s, %s) - """ % (mod, new, mod, new) - self.check(b, a) - -class Test_unicode(FixerTestCase): - fixer = "unicode" - - def test_whitespace(self): - b = """unicode( x)""" - a = """str( x)""" - self.check(b, a) - - b = """ unicode(x )""" - a = """ str(x )""" - self.check(b, a) - - b = """ u'h'""" - a = """ 'h'""" - self.check(b, a) - - def test_unicode_call(self): - b = """unicode(x, y, z)""" - a = """str(x, y, z)""" - self.check(b, a) - - def test_unichr(self): - b = """unichr(u'h')""" - a = """chr('h')""" - self.check(b, a) - - def test_unicode_literal_1(self): - b = '''u"x"''' - a = '''"x"''' - self.check(b, a) - - def test_unicode_literal_2(self): - b = """ur'x'""" - a = """r'x'""" - self.check(b, a) - - def test_unicode_literal_3(self): - b = """UR'''x''' """ - a = """R'''x''' """ - self.check(b, a) - - def test_native_literal_escape_u(self): - b = r"""'\\\u20ac\U0001d121\\u20ac'""" - a = r"""'\\\\u20ac\\U0001d121\\u20ac'""" - self.check(b, a) - - b = r"""r'\\\u20ac\U0001d121\\u20ac'""" - a = r"""r'\\\u20ac\U0001d121\\u20ac'""" - self.check(b, a) - - def test_bytes_literal_escape_u(self): - b = r"""b'\\\u20ac\U0001d121\\u20ac'""" - a = r"""b'\\\u20ac\U0001d121\\u20ac'""" - self.check(b, a) - - b = r"""br'\\\u20ac\U0001d121\\u20ac'""" - a = r"""br'\\\u20ac\U0001d121\\u20ac'""" - self.check(b, a) - - def test_unicode_literal_escape_u(self): - b = r"""u'\\\u20ac\U0001d121\\u20ac'""" - a = r"""'\\\u20ac\U0001d121\\u20ac'""" - self.check(b, a) - - b = r"""ur'\\\u20ac\U0001d121\\u20ac'""" - a = r"""r'\\\u20ac\U0001d121\\u20ac'""" - self.check(b, a) - - def test_native_unicode_literal_escape_u(self): - f = 'from __future__ import unicode_literals\n' - b = f + r"""'\\\u20ac\U0001d121\\u20ac'""" - a = f + r"""'\\\u20ac\U0001d121\\u20ac'""" - self.check(b, a) - - b = f + r"""r'\\\u20ac\U0001d121\\u20ac'""" - a = f + r"""r'\\\u20ac\U0001d121\\u20ac'""" - self.check(b, a) - - -class Test_filter(FixerTestCase): - fixer = "filter" - - def test_prefix_preservation(self): - b = """x = filter( foo, 'abc' )""" - a = """x = list(filter( foo, 'abc' ))""" - self.check(b, a) - - b = """x = filter( None , 'abc' )""" - a = """x = [_f for _f in 'abc' if _f]""" - self.check(b, a) - - def test_filter_basic(self): - b = """x = filter(None, 'abc')""" - a = """x = [_f for _f in 'abc' if _f]""" - self.check(b, a) - - b = """x = len(filter(f, 'abc'))""" - a = """x = len(list(filter(f, 'abc')))""" - self.check(b, a) - - b = """x = filter(lambda x: x%2 == 0, range(10))""" - a = """x = [x for x in range(10) if x%2 == 0]""" - self.check(b, a) - - # Note the parens around x - b = """x = filter(lambda (x): x%2 == 0, range(10))""" - a = """x = [x for x in range(10) if x%2 == 0]""" - self.check(b, a) - - # bpo-38871 - b = """filter(lambda x: True if x > 2 else False, [1, 2, 3])""" - a = """[x for x in [1, 2, 3] if (True if x > 2 else False)]""" - self.check(b, a) - - def test_filter_trailers(self): - b = """x = filter(None, 'abc')[0]""" - a = """x = [_f for _f in 'abc' if _f][0]""" - self.check(b, a) - - b = """x = len(filter(f, 'abc')[0])""" - a = """x = len(list(filter(f, 'abc'))[0])""" - self.check(b, a) - - b = """x = filter(lambda x: x%2 == 0, range(10))[0]""" - a = """x = [x for x in range(10) if x%2 == 0][0]""" - self.check(b, a) - - # Note the parens around x - b = """x = filter(lambda (x): x%2 == 0, range(10))[0]""" - a = """x = [x for x in range(10) if x%2 == 0][0]""" - self.check(b, a) - - def test_filter_nochange(self): - a = """b.join(filter(f, 'abc'))""" - self.unchanged(a) - a = """(a + foo(5)).join(filter(f, 'abc'))""" - self.unchanged(a) - a = """iter(filter(f, 'abc'))""" - self.unchanged(a) - a = """list(filter(f, 'abc'))""" - self.unchanged(a) - a = """list(filter(f, 'abc'))[0]""" - self.unchanged(a) - a = """set(filter(f, 'abc'))""" - self.unchanged(a) - a = """set(filter(f, 'abc')).pop()""" - self.unchanged(a) - a = """tuple(filter(f, 'abc'))""" - self.unchanged(a) - a = """any(filter(f, 'abc'))""" - self.unchanged(a) - a = """all(filter(f, 'abc'))""" - self.unchanged(a) - a = """sum(filter(f, 'abc'))""" - self.unchanged(a) - a = """sorted(filter(f, 'abc'))""" - self.unchanged(a) - a = """sorted(filter(f, 'abc'), key=blah)""" - self.unchanged(a) - a = """sorted(filter(f, 'abc'), key=blah)[0]""" - self.unchanged(a) - a = """enumerate(filter(f, 'abc'))""" - self.unchanged(a) - a = """enumerate(filter(f, 'abc'), start=1)""" - self.unchanged(a) - a = """for i in filter(f, 'abc'): pass""" - self.unchanged(a) - a = """[x for x in filter(f, 'abc')]""" - self.unchanged(a) - a = """(x for x in filter(f, 'abc'))""" - self.unchanged(a) - - def test_future_builtins(self): - a = "from future_builtins import spam, filter; filter(f, 'ham')" - self.unchanged(a) - - b = """from future_builtins import spam; x = filter(f, 'abc')""" - a = """from future_builtins import spam; x = list(filter(f, 'abc'))""" - self.check(b, a) - - a = "from future_builtins import *; filter(f, 'ham')" - self.unchanged(a) - -class Test_map(FixerTestCase): - fixer = "map" - - def check(self, b, a): - self.unchanged("from future_builtins import map; " + b, a) - super(Test_map, self).check(b, a) - - def test_prefix_preservation(self): - b = """x = map( f, 'abc' )""" - a = """x = list(map( f, 'abc' ))""" - self.check(b, a) - - def test_map_trailers(self): - b = """x = map(f, 'abc')[0]""" - a = """x = list(map(f, 'abc'))[0]""" - self.check(b, a) - - b = """x = map(None, l)[0]""" - a = """x = list(l)[0]""" - self.check(b, a) - - b = """x = map(lambda x:x, l)[0]""" - a = """x = [x for x in l][0]""" - self.check(b, a) - - b = """x = map(f, 'abc')[0][1]""" - a = """x = list(map(f, 'abc'))[0][1]""" - self.check(b, a) - - def test_trailing_comment(self): - b = """x = map(f, 'abc') # foo""" - a = """x = list(map(f, 'abc')) # foo""" - self.check(b, a) - - def test_None_with_multiple_arguments(self): - s = """x = map(None, a, b, c)""" - self.warns_unchanged(s, "cannot convert map(None, ...) with " - "multiple arguments") - - def test_map_basic(self): - b = """x = map(f, 'abc')""" - a = """x = list(map(f, 'abc'))""" - self.check(b, a) - - b = """x = len(map(f, 'abc', 'def'))""" - a = """x = len(list(map(f, 'abc', 'def')))""" - self.check(b, a) - - b = """x = map(None, 'abc')""" - a = """x = list('abc')""" - self.check(b, a) - - b = """x = map(lambda x: x+1, range(4))""" - a = """x = [x+1 for x in range(4)]""" - self.check(b, a) - - # Note the parens around x - b = """x = map(lambda (x): x+1, range(4))""" - a = """x = [x+1 for x in range(4)]""" - self.check(b, a) - - b = """ - foo() - # foo - map(f, x) - """ - a = """ - foo() - # foo - list(map(f, x)) - """ - self.warns(b, a, "You should use a for loop here") - - def test_map_nochange(self): - a = """b.join(map(f, 'abc'))""" - self.unchanged(a) - a = """(a + foo(5)).join(map(f, 'abc'))""" - self.unchanged(a) - a = """iter(map(f, 'abc'))""" - self.unchanged(a) - a = """list(map(f, 'abc'))""" - self.unchanged(a) - a = """list(map(f, 'abc'))[0]""" - self.unchanged(a) - a = """set(map(f, 'abc'))""" - self.unchanged(a) - a = """set(map(f, 'abc')).pop()""" - self.unchanged(a) - a = """tuple(map(f, 'abc'))""" - self.unchanged(a) - a = """any(map(f, 'abc'))""" - self.unchanged(a) - a = """all(map(f, 'abc'))""" - self.unchanged(a) - a = """sum(map(f, 'abc'))""" - self.unchanged(a) - a = """sorted(map(f, 'abc'))""" - self.unchanged(a) - a = """sorted(map(f, 'abc'), key=blah)""" - self.unchanged(a) - a = """sorted(map(f, 'abc'), key=blah)[0]""" - self.unchanged(a) - a = """enumerate(map(f, 'abc'))""" - self.unchanged(a) - a = """enumerate(map(f, 'abc'), start=1)""" - self.unchanged(a) - a = """for i in map(f, 'abc'): pass""" - self.unchanged(a) - a = """[x for x in map(f, 'abc')]""" - self.unchanged(a) - a = """(x for x in map(f, 'abc'))""" - self.unchanged(a) - - def test_future_builtins(self): - a = "from future_builtins import spam, map, eggs; map(f, 'ham')" - self.unchanged(a) - - b = """from future_builtins import spam, eggs; x = map(f, 'abc')""" - a = """from future_builtins import spam, eggs; x = list(map(f, 'abc'))""" - self.check(b, a) - - a = "from future_builtins import *; map(f, 'ham')" - self.unchanged(a) - -class Test_zip(FixerTestCase): - fixer = "zip" - - def check(self, b, a): - self.unchanged("from future_builtins import zip; " + b, a) - super(Test_zip, self).check(b, a) - - def test_zip_basic(self): - b = """x = zip()""" - a = """x = list(zip())""" - self.check(b, a) - - b = """x = zip(a, b, c)""" - a = """x = list(zip(a, b, c))""" - self.check(b, a) - - b = """x = len(zip(a, b))""" - a = """x = len(list(zip(a, b)))""" - self.check(b, a) - - def test_zip_trailers(self): - b = """x = zip(a, b, c)[0]""" - a = """x = list(zip(a, b, c))[0]""" - self.check(b, a) - - b = """x = zip(a, b, c)[0][1]""" - a = """x = list(zip(a, b, c))[0][1]""" - self.check(b, a) - - def test_zip_nochange(self): - a = """b.join(zip(a, b))""" - self.unchanged(a) - a = """(a + foo(5)).join(zip(a, b))""" - self.unchanged(a) - a = """iter(zip(a, b))""" - self.unchanged(a) - a = """list(zip(a, b))""" - self.unchanged(a) - a = """list(zip(a, b))[0]""" - self.unchanged(a) - a = """set(zip(a, b))""" - self.unchanged(a) - a = """set(zip(a, b)).pop()""" - self.unchanged(a) - a = """tuple(zip(a, b))""" - self.unchanged(a) - a = """any(zip(a, b))""" - self.unchanged(a) - a = """all(zip(a, b))""" - self.unchanged(a) - a = """sum(zip(a, b))""" - self.unchanged(a) - a = """sorted(zip(a, b))""" - self.unchanged(a) - a = """sorted(zip(a, b), key=blah)""" - self.unchanged(a) - a = """sorted(zip(a, b), key=blah)[0]""" - self.unchanged(a) - a = """enumerate(zip(a, b))""" - self.unchanged(a) - a = """enumerate(zip(a, b), start=1)""" - self.unchanged(a) - a = """for i in zip(a, b): pass""" - self.unchanged(a) - a = """[x for x in zip(a, b)]""" - self.unchanged(a) - a = """(x for x in zip(a, b))""" - self.unchanged(a) - - def test_future_builtins(self): - a = "from future_builtins import spam, zip, eggs; zip(a, b)" - self.unchanged(a) - - b = """from future_builtins import spam, eggs; x = zip(a, b)""" - a = """from future_builtins import spam, eggs; x = list(zip(a, b))""" - self.check(b, a) - - a = "from future_builtins import *; zip(a, b)" - self.unchanged(a) - -class Test_standarderror(FixerTestCase): - fixer = "standarderror" - - def test(self): - b = """x = StandardError()""" - a = """x = Exception()""" - self.check(b, a) - - b = """x = StandardError(a, b, c)""" - a = """x = Exception(a, b, c)""" - self.check(b, a) - - b = """f(2 + StandardError(a, b, c))""" - a = """f(2 + Exception(a, b, c))""" - self.check(b, a) - -class Test_types(FixerTestCase): - fixer = "types" - - def test_basic_types_convert(self): - b = """types.StringType""" - a = """bytes""" - self.check(b, a) - - b = """types.DictType""" - a = """dict""" - self.check(b, a) - - b = """types . IntType""" - a = """int""" - self.check(b, a) - - b = """types.ListType""" - a = """list""" - self.check(b, a) - - b = """types.LongType""" - a = """int""" - self.check(b, a) - - b = """types.NoneType""" - a = """type(None)""" - self.check(b, a) - - b = "types.StringTypes" - a = "(str,)" - self.check(b, a) - -class Test_idioms(FixerTestCase): - fixer = "idioms" - - def test_while(self): - b = """while 1: foo()""" - a = """while True: foo()""" - self.check(b, a) - - b = """while 1: foo()""" - a = """while True: foo()""" - self.check(b, a) - - b = """ - while 1: - foo() - """ - a = """ - while True: - foo() - """ - self.check(b, a) - - def test_while_unchanged(self): - s = """while 11: foo()""" - self.unchanged(s) - - s = """while 0: foo()""" - self.unchanged(s) - - s = """while foo(): foo()""" - self.unchanged(s) - - s = """while []: foo()""" - self.unchanged(s) - - def test_eq_simple(self): - b = """type(x) == T""" - a = """isinstance(x, T)""" - self.check(b, a) - - b = """if type(x) == T: pass""" - a = """if isinstance(x, T): pass""" - self.check(b, a) - - def test_eq_reverse(self): - b = """T == type(x)""" - a = """isinstance(x, T)""" - self.check(b, a) - - b = """if T == type(x): pass""" - a = """if isinstance(x, T): pass""" - self.check(b, a) - - def test_eq_expression(self): - b = """type(x+y) == d.get('T')""" - a = """isinstance(x+y, d.get('T'))""" - self.check(b, a) - - b = """type( x + y) == d.get('T')""" - a = """isinstance(x + y, d.get('T'))""" - self.check(b, a) - - def test_is_simple(self): - b = """type(x) is T""" - a = """isinstance(x, T)""" - self.check(b, a) - - b = """if type(x) is T: pass""" - a = """if isinstance(x, T): pass""" - self.check(b, a) - - def test_is_reverse(self): - b = """T is type(x)""" - a = """isinstance(x, T)""" - self.check(b, a) - - b = """if T is type(x): pass""" - a = """if isinstance(x, T): pass""" - self.check(b, a) - - def test_is_expression(self): - b = """type(x+y) is d.get('T')""" - a = """isinstance(x+y, d.get('T'))""" - self.check(b, a) - - b = """type( x + y) is d.get('T')""" - a = """isinstance(x + y, d.get('T'))""" - self.check(b, a) - - def test_is_not_simple(self): - b = """type(x) is not T""" - a = """not isinstance(x, T)""" - self.check(b, a) - - b = """if type(x) is not T: pass""" - a = """if not isinstance(x, T): pass""" - self.check(b, a) - - def test_is_not_reverse(self): - b = """T is not type(x)""" - a = """not isinstance(x, T)""" - self.check(b, a) - - b = """if T is not type(x): pass""" - a = """if not isinstance(x, T): pass""" - self.check(b, a) - - def test_is_not_expression(self): - b = """type(x+y) is not d.get('T')""" - a = """not isinstance(x+y, d.get('T'))""" - self.check(b, a) - - b = """type( x + y) is not d.get('T')""" - a = """not isinstance(x + y, d.get('T'))""" - self.check(b, a) - - def test_ne_simple(self): - b = """type(x) != T""" - a = """not isinstance(x, T)""" - self.check(b, a) - - b = """if type(x) != T: pass""" - a = """if not isinstance(x, T): pass""" - self.check(b, a) - - def test_ne_reverse(self): - b = """T != type(x)""" - a = """not isinstance(x, T)""" - self.check(b, a) - - b = """if T != type(x): pass""" - a = """if not isinstance(x, T): pass""" - self.check(b, a) - - def test_ne_expression(self): - b = """type(x+y) != d.get('T')""" - a = """not isinstance(x+y, d.get('T'))""" - self.check(b, a) - - b = """type( x + y) != d.get('T')""" - a = """not isinstance(x + y, d.get('T'))""" - self.check(b, a) - - def test_type_unchanged(self): - a = """type(x).__name__""" - self.unchanged(a) - - def test_sort_list_call(self): - b = """ - v = list(t) - v.sort() - foo(v) - """ - a = """ - v = sorted(t) - foo(v) - """ - self.check(b, a) - - b = """ - v = list(foo(b) + d) - v.sort() - foo(v) - """ - a = """ - v = sorted(foo(b) + d) - foo(v) - """ - self.check(b, a) - - b = """ - while x: - v = list(t) - v.sort() - foo(v) - """ - a = """ - while x: - v = sorted(t) - foo(v) - """ - self.check(b, a) - - b = """ - v = list(t) - # foo - v.sort() - foo(v) - """ - a = """ - v = sorted(t) - # foo - foo(v) - """ - self.check(b, a) - - b = r""" - v = list( t) - v.sort() - foo(v) - """ - a = r""" - v = sorted( t) - foo(v) - """ - self.check(b, a) - - b = r""" - try: - m = list(s) - m.sort() - except: pass - """ - - a = r""" - try: - m = sorted(s) - except: pass - """ - self.check(b, a) - - b = r""" - try: - m = list(s) - # foo - m.sort() - except: pass - """ - - a = r""" - try: - m = sorted(s) - # foo - except: pass - """ - self.check(b, a) - - b = r""" - m = list(s) - # more comments - m.sort()""" - - a = r""" - m = sorted(s) - # more comments""" - self.check(b, a) - - def test_sort_simple_expr(self): - b = """ - v = t - v.sort() - foo(v) - """ - a = """ - v = sorted(t) - foo(v) - """ - self.check(b, a) - - b = """ - v = foo(b) - v.sort() - foo(v) - """ - a = """ - v = sorted(foo(b)) - foo(v) - """ - self.check(b, a) - - b = """ - v = b.keys() - v.sort() - foo(v) - """ - a = """ - v = sorted(b.keys()) - foo(v) - """ - self.check(b, a) - - b = """ - v = foo(b) + d - v.sort() - foo(v) - """ - a = """ - v = sorted(foo(b) + d) - foo(v) - """ - self.check(b, a) - - b = """ - while x: - v = t - v.sort() - foo(v) - """ - a = """ - while x: - v = sorted(t) - foo(v) - """ - self.check(b, a) - - b = """ - v = t - # foo - v.sort() - foo(v) - """ - a = """ - v = sorted(t) - # foo - foo(v) - """ - self.check(b, a) - - b = r""" - v = t - v.sort() - foo(v) - """ - a = r""" - v = sorted(t) - foo(v) - """ - self.check(b, a) - - def test_sort_unchanged(self): - s = """ - v = list(t) - w.sort() - foo(w) - """ - self.unchanged(s) - - s = """ - v = list(t) - v.sort(u) - foo(v) - """ - self.unchanged(s) - -class Test_basestring(FixerTestCase): - fixer = "basestring" - - def test_basestring(self): - b = """isinstance(x, basestring)""" - a = """isinstance(x, str)""" - self.check(b, a) - -class Test_buffer(FixerTestCase): - fixer = "buffer" - - def test_buffer(self): - b = """x = buffer(y)""" - a = """x = memoryview(y)""" - self.check(b, a) - - def test_slicing(self): - b = """buffer(y)[4:5]""" - a = """memoryview(y)[4:5]""" - self.check(b, a) - -class Test_future(FixerTestCase): - fixer = "future" - - def test_future(self): - b = """from __future__ import braces""" - a = """""" - self.check(b, a) - - b = """# comment\nfrom __future__ import braces""" - a = """# comment\n""" - self.check(b, a) - - b = """from __future__ import braces\n# comment""" - a = """\n# comment""" - self.check(b, a) - - def test_run_order(self): - self.assert_runs_after('print') - -class Test_itertools(FixerTestCase): - fixer = "itertools" - - def checkall(self, before, after): - # Because we need to check with and without the itertools prefix - # and on each of the three functions, these loops make it all - # much easier - for i in ('itertools.', ''): - for f in ('map', 'filter', 'zip'): - b = before %(i+'i'+f) - a = after %(f) - self.check(b, a) - - def test_0(self): - # A simple example -- test_1 covers exactly the same thing, - # but it's not quite as clear. - b = "itertools.izip(a, b)" - a = "zip(a, b)" - self.check(b, a) - - def test_1(self): - b = """%s(f, a)""" - a = """%s(f, a)""" - self.checkall(b, a) - - def test_qualified(self): - b = """itertools.ifilterfalse(a, b)""" - a = """itertools.filterfalse(a, b)""" - self.check(b, a) - - b = """itertools.izip_longest(a, b)""" - a = """itertools.zip_longest(a, b)""" - self.check(b, a) - - def test_2(self): - b = """ifilterfalse(a, b)""" - a = """filterfalse(a, b)""" - self.check(b, a) - - b = """izip_longest(a, b)""" - a = """zip_longest(a, b)""" - self.check(b, a) - - def test_space_1(self): - b = """ %s(f, a)""" - a = """ %s(f, a)""" - self.checkall(b, a) - - def test_space_2(self): - b = """ itertools.ifilterfalse(a, b)""" - a = """ itertools.filterfalse(a, b)""" - self.check(b, a) - - b = """ itertools.izip_longest(a, b)""" - a = """ itertools.zip_longest(a, b)""" - self.check(b, a) - - def test_run_order(self): - self.assert_runs_after('map', 'zip', 'filter') - - -class Test_itertools_imports(FixerTestCase): - fixer = 'itertools_imports' - - def test_reduced(self): - b = "from itertools import imap, izip, foo" - a = "from itertools import foo" - self.check(b, a) - - b = "from itertools import bar, imap, izip, foo" - a = "from itertools import bar, foo" - self.check(b, a) - - b = "from itertools import chain, imap, izip" - a = "from itertools import chain" - self.check(b, a) - - def test_comments(self): - b = "#foo\nfrom itertools import imap, izip" - a = "#foo\n" - self.check(b, a) - - def test_none(self): - b = "from itertools import imap, izip" - a = "" - self.check(b, a) - - b = "from itertools import izip" - a = "" - self.check(b, a) - - def test_import_as(self): - b = "from itertools import izip, bar as bang, imap" - a = "from itertools import bar as bang" - self.check(b, a) - - b = "from itertools import izip as _zip, imap, bar" - a = "from itertools import bar" - self.check(b, a) - - b = "from itertools import imap as _map" - a = "" - self.check(b, a) - - b = "from itertools import imap as _map, izip as _zip" - a = "" - self.check(b, a) - - s = "from itertools import bar as bang" - self.unchanged(s) - - def test_ifilter_and_zip_longest(self): - for name in "filterfalse", "zip_longest": - b = "from itertools import i%s" % (name,) - a = "from itertools import %s" % (name,) - self.check(b, a) - - b = "from itertools import imap, i%s, foo" % (name,) - a = "from itertools import %s, foo" % (name,) - self.check(b, a) - - b = "from itertools import bar, i%s, foo" % (name,) - a = "from itertools import bar, %s, foo" % (name,) - self.check(b, a) - - def test_import_star(self): - s = "from itertools import *" - self.unchanged(s) - - - def test_unchanged(self): - s = "from itertools import foo" - self.unchanged(s) - - -class Test_import(FixerTestCase): - fixer = "import" - - def setUp(self): - super(Test_import, self).setUp() - # Need to replace fix_import's exists method - # so we can check that it's doing the right thing - self.files_checked = [] - self.present_files = set() - self.always_exists = True - def fake_exists(name): - self.files_checked.append(name) - return self.always_exists or (name in self.present_files) - - from lib2to3.fixes import fix_import - fix_import.exists = fake_exists - - def tearDown(self): - from lib2to3.fixes import fix_import - fix_import.exists = os.path.exists - - def check_both(self, b, a): - self.always_exists = True - super(Test_import, self).check(b, a) - self.always_exists = False - super(Test_import, self).unchanged(b) - - def test_files_checked(self): - def p(path): - # Takes a unix path and returns a path with correct separators - return os.path.pathsep.join(path.split("/")) - - self.always_exists = False - self.present_files = set(['__init__.py']) - expected_extensions = ('.py', os.path.sep, '.pyc', '.so', '.sl', '.pyd') - names_to_test = (p("/spam/eggs.py"), "ni.py", p("../../shrubbery.py")) - - for name in names_to_test: - self.files_checked = [] - self.filename = name - self.unchanged("import jam") - - if os.path.dirname(name): - name = os.path.dirname(name) + '/jam' - else: - name = 'jam' - expected_checks = set(name + ext for ext in expected_extensions) - expected_checks.add("__init__.py") - - self.assertEqual(set(self.files_checked), expected_checks) - - def test_not_in_package(self): - s = "import bar" - self.always_exists = False - self.present_files = set(["bar.py"]) - self.unchanged(s) - - def test_with_absolute_import_enabled(self): - s = "from __future__ import absolute_import\nimport bar" - self.always_exists = False - self.present_files = set(["__init__.py", "bar.py"]) - self.unchanged(s) - - def test_in_package(self): - b = "import bar" - a = "from . import bar" - self.always_exists = False - self.present_files = set(["__init__.py", "bar.py"]) - self.check(b, a) - - def test_import_from_package(self): - b = "import bar" - a = "from . import bar" - self.always_exists = False - self.present_files = set(["__init__.py", "bar" + os.path.sep]) - self.check(b, a) - - def test_already_relative_import(self): - s = "from . import bar" - self.unchanged(s) - - def test_comments_and_indent(self): - b = "import bar # Foo" - a = "from . import bar # Foo" - self.check(b, a) - - def test_from(self): - b = "from foo import bar, baz" - a = "from .foo import bar, baz" - self.check_both(b, a) - - b = "from foo import bar" - a = "from .foo import bar" - self.check_both(b, a) - - b = "from foo import (bar, baz)" - a = "from .foo import (bar, baz)" - self.check_both(b, a) - - def test_dotted_from(self): - b = "from green.eggs import ham" - a = "from .green.eggs import ham" - self.check_both(b, a) - - def test_from_as(self): - b = "from green.eggs import ham as spam" - a = "from .green.eggs import ham as spam" - self.check_both(b, a) - - def test_import(self): - b = "import foo" - a = "from . import foo" - self.check_both(b, a) - - b = "import foo, bar" - a = "from . import foo, bar" - self.check_both(b, a) - - b = "import foo, bar, x" - a = "from . import foo, bar, x" - self.check_both(b, a) - - b = "import x, y, z" - a = "from . import x, y, z" - self.check_both(b, a) - - def test_import_as(self): - b = "import foo as x" - a = "from . import foo as x" - self.check_both(b, a) - - b = "import a as b, b as c, c as d" - a = "from . import a as b, b as c, c as d" - self.check_both(b, a) - - def test_local_and_absolute(self): - self.always_exists = False - self.present_files = set(["foo.py", "__init__.py"]) - - s = "import foo, bar" - self.warns_unchanged(s, "absolute and local imports together") - - def test_dotted_import(self): - b = "import foo.bar" - a = "from . import foo.bar" - self.check_both(b, a) - - def test_dotted_import_as(self): - b = "import foo.bar as bang" - a = "from . import foo.bar as bang" - self.check_both(b, a) - - def test_prefix(self): - b = """ - # prefix - import foo.bar - """ - a = """ - # prefix - from . import foo.bar - """ - self.check_both(b, a) - - -class Test_set_literal(FixerTestCase): - - fixer = "set_literal" - - def test_basic(self): - b = """set([1, 2, 3])""" - a = """{1, 2, 3}""" - self.check(b, a) - - b = """set((1, 2, 3))""" - a = """{1, 2, 3}""" - self.check(b, a) - - b = """set((1,))""" - a = """{1}""" - self.check(b, a) - - b = """set([1])""" - self.check(b, a) - - b = """set((a, b))""" - a = """{a, b}""" - self.check(b, a) - - b = """set([a, b])""" - self.check(b, a) - - b = """set((a*234, f(args=23)))""" - a = """{a*234, f(args=23)}""" - self.check(b, a) - - b = """set([a*23, f(23)])""" - a = """{a*23, f(23)}""" - self.check(b, a) - - b = """set([a-234**23])""" - a = """{a-234**23}""" - self.check(b, a) - - def test_listcomps(self): - b = """set([x for x in y])""" - a = """{x for x in y}""" - self.check(b, a) - - b = """set([x for x in y if x == m])""" - a = """{x for x in y if x == m}""" - self.check(b, a) - - b = """set([x for x in y for a in b])""" - a = """{x for x in y for a in b}""" - self.check(b, a) - - b = """set([f(x) - 23 for x in y])""" - a = """{f(x) - 23 for x in y}""" - self.check(b, a) - - def test_whitespace(self): - b = """set( [1, 2])""" - a = """{1, 2}""" - self.check(b, a) - - b = """set([1 , 2])""" - a = """{1 , 2}""" - self.check(b, a) - - b = """set([ 1 ])""" - a = """{ 1 }""" - self.check(b, a) - - b = """set( [1] )""" - a = """{1}""" - self.check(b, a) - - b = """set([ 1, 2 ])""" - a = """{ 1, 2 }""" - self.check(b, a) - - b = """set([x for x in y ])""" - a = """{x for x in y }""" - self.check(b, a) - - b = """set( - [1, 2] - ) - """ - a = """{1, 2}\n""" - self.check(b, a) - - def test_comments(self): - b = """set((1, 2)) # Hi""" - a = """{1, 2} # Hi""" - self.check(b, a) - - # This isn't optimal behavior, but the fixer is optional. - b = """ - # Foo - set( # Bar - (1, 2) - ) - """ - a = """ - # Foo - {1, 2} - """ - self.check(b, a) - - def test_unchanged(self): - s = """set()""" - self.unchanged(s) - - s = """set(a)""" - self.unchanged(s) - - s = """set(a, b, c)""" - self.unchanged(s) - - # Don't transform generators because they might have to be lazy. - s = """set(x for x in y)""" - self.unchanged(s) - - s = """set(x for x in y if z)""" - self.unchanged(s) - - s = """set(a*823-23**2 + f(23))""" - self.unchanged(s) - - -class Test_sys_exc(FixerTestCase): - fixer = "sys_exc" - - def test_0(self): - b = "sys.exc_type" - a = "sys.exc_info()[0]" - self.check(b, a) - - def test_1(self): - b = "sys.exc_value" - a = "sys.exc_info()[1]" - self.check(b, a) - - def test_2(self): - b = "sys.exc_traceback" - a = "sys.exc_info()[2]" - self.check(b, a) - - def test_3(self): - b = "sys.exc_type # Foo" - a = "sys.exc_info()[0] # Foo" - self.check(b, a) - - def test_4(self): - b = "sys. exc_type" - a = "sys. exc_info()[0]" - self.check(b, a) - - def test_5(self): - b = "sys .exc_type" - a = "sys .exc_info()[0]" - self.check(b, a) - - -class Test_paren(FixerTestCase): - fixer = "paren" - - def test_0(self): - b = """[i for i in 1, 2 ]""" - a = """[i for i in (1, 2) ]""" - self.check(b, a) - - def test_1(self): - b = """[i for i in 1, 2, ]""" - a = """[i for i in (1, 2,) ]""" - self.check(b, a) - - def test_2(self): - b = """[i for i in 1, 2 ]""" - a = """[i for i in (1, 2) ]""" - self.check(b, a) - - def test_3(self): - b = """[i for i in 1, 2 if i]""" - a = """[i for i in (1, 2) if i]""" - self.check(b, a) - - def test_4(self): - b = """[i for i in 1, 2 ]""" - a = """[i for i in (1, 2) ]""" - self.check(b, a) - - def test_5(self): - b = """(i for i in 1, 2)""" - a = """(i for i in (1, 2))""" - self.check(b, a) - - def test_6(self): - b = """(i for i in 1 ,2 if i)""" - a = """(i for i in (1 ,2) if i)""" - self.check(b, a) - - def test_unchanged_0(self): - s = """[i for i in (1, 2)]""" - self.unchanged(s) - - def test_unchanged_1(self): - s = """[i for i in foo()]""" - self.unchanged(s) - - def test_unchanged_2(self): - s = """[i for i in (1, 2) if nothing]""" - self.unchanged(s) - - def test_unchanged_3(self): - s = """(i for i in (1, 2))""" - self.unchanged(s) - - def test_unchanged_4(self): - s = """[i for i in m]""" - self.unchanged(s) - -class Test_metaclass(FixerTestCase): - - fixer = 'metaclass' - - def test_unchanged(self): - self.unchanged("class X(): pass") - self.unchanged("class X(object): pass") - self.unchanged("class X(object1, object2): pass") - self.unchanged("class X(object1, object2, object3): pass") - self.unchanged("class X(metaclass=Meta): pass") - self.unchanged("class X(b, arg=23, metclass=Meta): pass") - self.unchanged("class X(b, arg=23, metaclass=Meta, other=42): pass") - - s = """ - class X: - def __metaclass__(self): pass - """ - self.unchanged(s) - - s = """ - class X: - a[23] = 74 - """ - self.unchanged(s) - - def test_comments(self): - b = """ - class X: - # hi - __metaclass__ = AppleMeta - """ - a = """ - class X(metaclass=AppleMeta): - # hi - pass - """ - self.check(b, a) - - b = """ - class X: - __metaclass__ = Meta - # Bedtime! - """ - a = """ - class X(metaclass=Meta): - pass - # Bedtime! - """ - self.check(b, a) - - def test_meta(self): - # no-parent class, odd body - b = """ - class X(): - __metaclass__ = Q - pass - """ - a = """ - class X(metaclass=Q): - pass - """ - self.check(b, a) - - # one parent class, no body - b = """class X(object): __metaclass__ = Q""" - a = """class X(object, metaclass=Q): pass""" - self.check(b, a) - - - # one parent, simple body - b = """ - class X(object): - __metaclass__ = Meta - bar = 7 - """ - a = """ - class X(object, metaclass=Meta): - bar = 7 - """ - self.check(b, a) - - b = """ - class X: - __metaclass__ = Meta; x = 4; g = 23 - """ - a = """ - class X(metaclass=Meta): - x = 4; g = 23 - """ - self.check(b, a) - - # one parent, simple body, __metaclass__ last - b = """ - class X(object): - bar = 7 - __metaclass__ = Meta - """ - a = """ - class X(object, metaclass=Meta): - bar = 7 - """ - self.check(b, a) - - # redefining __metaclass__ - b = """ - class X(): - __metaclass__ = A - __metaclass__ = B - bar = 7 - """ - a = """ - class X(metaclass=B): - bar = 7 - """ - self.check(b, a) - - # multiple inheritance, simple body - b = """ - class X(clsA, clsB): - __metaclass__ = Meta - bar = 7 - """ - a = """ - class X(clsA, clsB, metaclass=Meta): - bar = 7 - """ - self.check(b, a) - - # keywords in the class statement - b = """class m(a, arg=23): __metaclass__ = Meta""" - a = """class m(a, arg=23, metaclass=Meta): pass""" - self.check(b, a) - - b = """ - class X(expression(2 + 4)): - __metaclass__ = Meta - """ - a = """ - class X(expression(2 + 4), metaclass=Meta): - pass - """ - self.check(b, a) - - b = """ - class X(expression(2 + 4), x**4): - __metaclass__ = Meta - """ - a = """ - class X(expression(2 + 4), x**4, metaclass=Meta): - pass - """ - self.check(b, a) - - b = """ - class X: - __metaclass__ = Meta - save.py = 23 - """ - a = """ - class X(metaclass=Meta): - save.py = 23 - """ - self.check(b, a) - - -class Test_getcwdu(FixerTestCase): - - fixer = 'getcwdu' - - def test_basic(self): - b = """os.getcwdu""" - a = """os.getcwd""" - self.check(b, a) - - b = """os.getcwdu()""" - a = """os.getcwd()""" - self.check(b, a) - - b = """meth = os.getcwdu""" - a = """meth = os.getcwd""" - self.check(b, a) - - b = """os.getcwdu(args)""" - a = """os.getcwd(args)""" - self.check(b, a) - - def test_comment(self): - b = """os.getcwdu() # Foo""" - a = """os.getcwd() # Foo""" - self.check(b, a) - - def test_unchanged(self): - s = """os.getcwd()""" - self.unchanged(s) - - s = """getcwdu()""" - self.unchanged(s) - - s = """os.getcwdb()""" - self.unchanged(s) - - def test_indentation(self): - b = """ - if 1: - os.getcwdu() - """ - a = """ - if 1: - os.getcwd() - """ - self.check(b, a) - - def test_multilation(self): - b = """os .getcwdu()""" - a = """os .getcwd()""" - self.check(b, a) - - b = """os. getcwdu""" - a = """os. getcwd""" - self.check(b, a) - - b = """os.getcwdu ( )""" - a = """os.getcwd ( )""" - self.check(b, a) - - -class Test_operator(FixerTestCase): - - fixer = "operator" - - def test_operator_isCallable(self): - b = "operator.isCallable(x)" - a = "callable(x)" - self.check(b, a) - - def test_operator_sequenceIncludes(self): - b = "operator.sequenceIncludes(x, y)" - a = "operator.contains(x, y)" - self.check(b, a) - - b = "operator .sequenceIncludes(x, y)" - a = "operator .contains(x, y)" - self.check(b, a) - - b = "operator. sequenceIncludes(x, y)" - a = "operator. contains(x, y)" - self.check(b, a) - - def test_operator_isSequenceType(self): - b = "operator.isSequenceType(x)" - a = "import collections.abc\nisinstance(x, collections.abc.Sequence)" - self.check(b, a) - - def test_operator_isMappingType(self): - b = "operator.isMappingType(x)" - a = "import collections.abc\nisinstance(x, collections.abc.Mapping)" - self.check(b, a) - - def test_operator_isNumberType(self): - b = "operator.isNumberType(x)" - a = "import numbers\nisinstance(x, numbers.Number)" - self.check(b, a) - - def test_operator_repeat(self): - b = "operator.repeat(x, n)" - a = "operator.mul(x, n)" - self.check(b, a) - - b = "operator .repeat(x, n)" - a = "operator .mul(x, n)" - self.check(b, a) - - b = "operator. repeat(x, n)" - a = "operator. mul(x, n)" - self.check(b, a) - - def test_operator_irepeat(self): - b = "operator.irepeat(x, n)" - a = "operator.imul(x, n)" - self.check(b, a) - - b = "operator .irepeat(x, n)" - a = "operator .imul(x, n)" - self.check(b, a) - - b = "operator. irepeat(x, n)" - a = "operator. imul(x, n)" - self.check(b, a) - - def test_bare_isCallable(self): - s = "isCallable(x)" - t = "You should use 'callable(x)' here." - self.warns_unchanged(s, t) - - def test_bare_sequenceIncludes(self): - s = "sequenceIncludes(x, y)" - t = "You should use 'operator.contains(x, y)' here." - self.warns_unchanged(s, t) - - def test_bare_operator_isSequenceType(self): - s = "isSequenceType(z)" - t = "You should use 'isinstance(z, collections.abc.Sequence)' here." - self.warns_unchanged(s, t) - - def test_bare_operator_isMappingType(self): - s = "isMappingType(x)" - t = "You should use 'isinstance(x, collections.abc.Mapping)' here." - self.warns_unchanged(s, t) - - def test_bare_operator_isNumberType(self): - s = "isNumberType(y)" - t = "You should use 'isinstance(y, numbers.Number)' here." - self.warns_unchanged(s, t) - - def test_bare_operator_repeat(self): - s = "repeat(x, n)" - t = "You should use 'operator.mul(x, n)' here." - self.warns_unchanged(s, t) - - def test_bare_operator_irepeat(self): - s = "irepeat(y, 187)" - t = "You should use 'operator.imul(y, 187)' here." - self.warns_unchanged(s, t) - - -class Test_exitfunc(FixerTestCase): - - fixer = "exitfunc" - - def test_simple(self): - b = """ - import sys - sys.exitfunc = my_atexit - """ - a = """ - import sys - import atexit - atexit.register(my_atexit) - """ - self.check(b, a) - - def test_names_import(self): - b = """ - import sys, crumbs - sys.exitfunc = my_func - """ - a = """ - import sys, crumbs, atexit - atexit.register(my_func) - """ - self.check(b, a) - - def test_complex_expression(self): - b = """ - import sys - sys.exitfunc = do(d)/a()+complex(f=23, g=23)*expression - """ - a = """ - import sys - import atexit - atexit.register(do(d)/a()+complex(f=23, g=23)*expression) - """ - self.check(b, a) - - def test_comments(self): - b = """ - import sys # Foo - sys.exitfunc = f # Blah - """ - a = """ - import sys - import atexit # Foo - atexit.register(f) # Blah - """ - self.check(b, a) - - b = """ - import apples, sys, crumbs, larry # Pleasant comments - sys.exitfunc = func - """ - a = """ - import apples, sys, crumbs, larry, atexit # Pleasant comments - atexit.register(func) - """ - self.check(b, a) - - def test_in_a_function(self): - b = """ - import sys - def f(): - sys.exitfunc = func - """ - a = """ - import sys - import atexit - def f(): - atexit.register(func) - """ - self.check(b, a) - - def test_no_sys_import(self): - b = """sys.exitfunc = f""" - a = """atexit.register(f)""" - msg = ("Can't find sys import; Please add an atexit import at the " - "top of your file.") - self.warns(b, a, msg) - - - def test_unchanged(self): - s = """f(sys.exitfunc)""" - self.unchanged(s) - - -class Test_asserts(FixerTestCase): - - fixer = "asserts" - - def test_deprecated_names(self): - tests = [ - ('self.assert_(True)', 'self.assertTrue(True)'), - ('self.assertEquals(2, 2)', 'self.assertEqual(2, 2)'), - ('self.assertNotEquals(2, 3)', 'self.assertNotEqual(2, 3)'), - ('self.assertAlmostEquals(2, 3)', 'self.assertAlmostEqual(2, 3)'), - ('self.assertNotAlmostEquals(2, 8)', 'self.assertNotAlmostEqual(2, 8)'), - ('self.failUnlessEqual(2, 2)', 'self.assertEqual(2, 2)'), - ('self.failIfEqual(2, 3)', 'self.assertNotEqual(2, 3)'), - ('self.failUnlessAlmostEqual(2, 3)', 'self.assertAlmostEqual(2, 3)'), - ('self.failIfAlmostEqual(2, 8)', 'self.assertNotAlmostEqual(2, 8)'), - ('self.failUnless(True)', 'self.assertTrue(True)'), - ('self.failUnlessRaises(foo)', 'self.assertRaises(foo)'), - ('self.failIf(False)', 'self.assertFalse(False)'), - ] - for b, a in tests: - self.check(b, a) - - def test_variants(self): - b = 'eq = self.assertEquals' - a = 'eq = self.assertEqual' - self.check(b, a) - b = 'self.assertEquals(2, 3, msg="fail")' - a = 'self.assertEqual(2, 3, msg="fail")' - self.check(b, a) - b = 'self.assertEquals(2, 3, msg="fail") # foo' - a = 'self.assertEqual(2, 3, msg="fail") # foo' - self.check(b, a) - b = 'self.assertEquals (2, 3)' - a = 'self.assertEqual (2, 3)' - self.check(b, a) - b = ' self.assertEquals (2, 3)' - a = ' self.assertEqual (2, 3)' - self.check(b, a) - b = 'with self.failUnlessRaises(Explosion): explode()' - a = 'with self.assertRaises(Explosion): explode()' - self.check(b, a) - b = 'with self.failUnlessRaises(Explosion) as cm: explode()' - a = 'with self.assertRaises(Explosion) as cm: explode()' - self.check(b, a) - - def test_unchanged(self): - self.unchanged('self.assertEqualsOnSaturday') - self.unchanged('self.assertEqualsOnSaturday(3, 5)') diff --git a/Lib/test/test_lib2to3/test_main.py b/Lib/test/test_lib2to3/test_main.py deleted file mode 100644 index a33c45c50a0de2..00000000000000 --- a/Lib/test/test_lib2to3/test_main.py +++ /dev/null @@ -1,139 +0,0 @@ -# -*- coding: utf-8 -*- -import codecs -import io -import logging -import os -import re -import shutil -import sys -import tempfile -import unittest - -from lib2to3 import main - - -TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "data") -PY2_TEST_MODULE = os.path.join(TEST_DATA_DIR, "py2_test_grammar.py") - - -class TestMain(unittest.TestCase): - - def setUp(self): - self.temp_dir = None # tearDown() will rmtree this directory if set. - - def tearDown(self): - # Clean up logging configuration down by main. - del logging.root.handlers[:] - if self.temp_dir: - shutil.rmtree(self.temp_dir) - - def run_2to3_capture(self, args, in_capture, out_capture, err_capture): - save_stdin = sys.stdin - save_stdout = sys.stdout - save_stderr = sys.stderr - sys.stdin = in_capture - sys.stdout = out_capture - sys.stderr = err_capture - try: - return main.main("lib2to3.fixes", args) - finally: - sys.stdin = save_stdin - sys.stdout = save_stdout - sys.stderr = save_stderr - - def test_unencodable_diff(self): - input_stream = io.StringIO("print 'nothing'\nprint u'über'\n") - out = io.BytesIO() - out_enc = codecs.getwriter("ascii")(out) - err = io.StringIO() - ret = self.run_2to3_capture(["-"], input_stream, out_enc, err) - self.assertEqual(ret, 0) - output = out.getvalue().decode("ascii") - self.assertIn("-print 'nothing'", output) - self.assertIn("WARNING: couldn't encode 's diff for " - "your terminal", err.getvalue()) - - def setup_test_source_trees(self): - """Setup a test source tree and output destination tree.""" - self.temp_dir = tempfile.mkdtemp() # tearDown() cleans this up. - self.py2_src_dir = os.path.join(self.temp_dir, "python2_project") - self.py3_dest_dir = os.path.join(self.temp_dir, "python3_project") - os.mkdir(self.py2_src_dir) - os.mkdir(self.py3_dest_dir) - # Turn it into a package with a few files. - self.setup_files = [] - open(os.path.join(self.py2_src_dir, "__init__.py"), "w").close() - self.setup_files.append("__init__.py") - shutil.copy(PY2_TEST_MODULE, self.py2_src_dir) - self.setup_files.append(os.path.basename(PY2_TEST_MODULE)) - self.trivial_py2_file = os.path.join(self.py2_src_dir, "trivial.py") - self.init_py2_file = os.path.join(self.py2_src_dir, "__init__.py") - with open(self.trivial_py2_file, "w") as trivial: - trivial.write("print 'I need a simple conversion.'") - self.setup_files.append("trivial.py") - - def test_filename_changing_on_output_single_dir(self): - """2to3 a single directory with a new output dir and suffix.""" - self.setup_test_source_trees() - out = io.StringIO() - err = io.StringIO() - suffix = "TEST" - ret = self.run_2to3_capture( - ["-n", "--add-suffix", suffix, "--write-unchanged-files", - "--no-diffs", "--output-dir", - self.py3_dest_dir, self.py2_src_dir], - io.StringIO(""), out, err) - self.assertEqual(ret, 0) - stderr = err.getvalue() - self.assertIn(" implies -w.", stderr) - self.assertIn( - "Output in %r will mirror the input directory %r layout" % ( - self.py3_dest_dir, self.py2_src_dir), stderr) - self.assertEqual(set(name+suffix for name in self.setup_files), - set(os.listdir(self.py3_dest_dir))) - for name in self.setup_files: - self.assertIn("Writing converted %s to %s" % ( - os.path.join(self.py2_src_dir, name), - os.path.join(self.py3_dest_dir, name+suffix)), stderr) - sep = re.escape(os.sep) - self.assertRegex( - stderr, r"No changes to .*/__init__\.py".replace("/", sep)) - self.assertNotRegex( - stderr, r"No changes to .*/trivial\.py".replace("/", sep)) - - def test_filename_changing_on_output_two_files(self): - """2to3 two files in one directory with a new output dir.""" - self.setup_test_source_trees() - err = io.StringIO() - py2_files = [self.trivial_py2_file, self.init_py2_file] - expected_files = set(os.path.basename(name) for name in py2_files) - ret = self.run_2to3_capture( - ["-n", "-w", "--write-unchanged-files", - "--no-diffs", "--output-dir", self.py3_dest_dir] + py2_files, - io.StringIO(""), io.StringIO(), err) - self.assertEqual(ret, 0) - stderr = err.getvalue() - self.assertIn( - "Output in %r will mirror the input directory %r layout" % ( - self.py3_dest_dir, self.py2_src_dir), stderr) - self.assertEqual(expected_files, set(os.listdir(self.py3_dest_dir))) - - def test_filename_changing_on_output_single_file(self): - """2to3 a single file with a new output dir.""" - self.setup_test_source_trees() - err = io.StringIO() - ret = self.run_2to3_capture( - ["-n", "-w", "--no-diffs", "--output-dir", self.py3_dest_dir, - self.trivial_py2_file], - io.StringIO(""), io.StringIO(), err) - self.assertEqual(ret, 0) - stderr = err.getvalue() - self.assertIn( - "Output in %r will mirror the input directory %r layout" % ( - self.py3_dest_dir, self.py2_src_dir), stderr) - self.assertEqual(set([os.path.basename(self.trivial_py2_file)]), - set(os.listdir(self.py3_dest_dir))) - - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/test/test_lib2to3/test_parser.py b/Lib/test/test_lib2to3/test_parser.py deleted file mode 100644 index 2c798b181fdbda..00000000000000 --- a/Lib/test/test_lib2to3/test_parser.py +++ /dev/null @@ -1,718 +0,0 @@ -"""Test suite for 2to3's parser and grammar files. - -This is the place to add tests for changes to 2to3's grammar, such as those -merging the grammars for Python 2 and 3. In addition to specific tests for -parts of the grammar we've changed, we also make sure we can parse the -test_grammar.py files from both Python 2 and Python 3. -""" - -# Testing imports -from . import support -from .support import driver, driver_no_print_statement - -# Python imports -import difflib -import importlib -import operator -import os -import pickle -import shutil -import subprocess -import sys -import tempfile -import test.support -import unittest - -# Local imports -from lib2to3.pgen2 import driver as pgen2_driver -from lib2to3.pgen2 import tokenize -from lib2to3.pgen2.parse import ParseError -from lib2to3.pygram import python_symbols as syms - - -class TestDriver(support.TestCase): - - def test_formfeed(self): - s = """print 1\n\x0Cprint 2\n""" - t = driver.parse_string(s) - self.assertEqual(t.children[0].children[0].type, syms.print_stmt) - self.assertEqual(t.children[1].children[0].type, syms.print_stmt) - - -class TestPgen2Caching(support.TestCase): - def test_load_grammar_from_txt_file(self): - pgen2_driver.load_grammar(support.grammar_path, save=False, force=True) - - def test_load_grammar_from_pickle(self): - # Make a copy of the grammar file in a temp directory we are - # guaranteed to be able to write to. - tmpdir = tempfile.mkdtemp() - try: - grammar_copy = os.path.join( - tmpdir, os.path.basename(support.grammar_path)) - shutil.copy(support.grammar_path, grammar_copy) - pickle_name = pgen2_driver._generate_pickle_name(grammar_copy) - - pgen2_driver.load_grammar(grammar_copy, save=True, force=True) - self.assertTrue(os.path.exists(pickle_name)) - - os.unlink(grammar_copy) # Only the pickle remains... - pgen2_driver.load_grammar(grammar_copy, save=False, force=False) - finally: - shutil.rmtree(tmpdir) - - @unittest.skipIf(sys.executable is None, 'sys.executable required') - @unittest.skipIf( - sys.platform in {'emscripten', 'wasi'}, 'requires working subprocess' - ) - def test_load_grammar_from_subprocess(self): - tmpdir = tempfile.mkdtemp() - tmpsubdir = os.path.join(tmpdir, 'subdir') - try: - os.mkdir(tmpsubdir) - grammar_base = os.path.basename(support.grammar_path) - grammar_copy = os.path.join(tmpdir, grammar_base) - grammar_sub_copy = os.path.join(tmpsubdir, grammar_base) - shutil.copy(support.grammar_path, grammar_copy) - shutil.copy(support.grammar_path, grammar_sub_copy) - pickle_name = pgen2_driver._generate_pickle_name(grammar_copy) - pickle_sub_name = pgen2_driver._generate_pickle_name( - grammar_sub_copy) - self.assertNotEqual(pickle_name, pickle_sub_name) - - # Generate a pickle file from this process. - pgen2_driver.load_grammar(grammar_copy, save=True, force=True) - self.assertTrue(os.path.exists(pickle_name)) - - # Generate a new pickle file in a subprocess with a most likely - # different hash randomization seed. - sub_env = dict(os.environ) - sub_env['PYTHONHASHSEED'] = 'random' - code = """ -from lib2to3.pgen2 import driver as pgen2_driver -pgen2_driver.load_grammar(%r, save=True, force=True) - """ % (grammar_sub_copy,) - cmd = [sys.executable, - '-Wignore:lib2to3:DeprecationWarning', - '-c', code] - subprocess.check_call( cmd, env=sub_env) - self.assertTrue(os.path.exists(pickle_sub_name)) - - with open(pickle_name, 'rb') as pickle_f_1, \ - open(pickle_sub_name, 'rb') as pickle_f_2: - self.assertEqual( - pickle_f_1.read(), pickle_f_2.read(), - msg='Grammar caches generated using different hash seeds' - ' were not identical.') - finally: - shutil.rmtree(tmpdir) - - def test_load_packaged_grammar(self): - modname = __name__ + '.load_test' - class MyLoader: - def get_data(self, where): - return pickle.dumps({'elephant': 19}) - class MyModule: - __file__ = 'parsertestmodule' - __spec__ = importlib.util.spec_from_loader(modname, MyLoader()) - sys.modules[modname] = MyModule() - self.addCleanup(operator.delitem, sys.modules, modname) - g = pgen2_driver.load_packaged_grammar(modname, 'Grammar.txt') - self.assertEqual(g.elephant, 19) - - -class GrammarTest(support.TestCase): - def validate(self, code): - support.parse_string(code) - - def invalid_syntax(self, code): - try: - self.validate(code) - except ParseError: - pass - else: - raise AssertionError("Syntax shouldn't have been valid") - - -class TestMatrixMultiplication(GrammarTest): - def test_matrix_multiplication_operator(self): - self.validate("a @ b") - self.validate("a @= b") - - -class TestYieldFrom(GrammarTest): - def test_yield_from(self): - self.validate("yield from x") - self.validate("(yield from x) + y") - self.invalid_syntax("yield from") - - -class TestAsyncAwait(GrammarTest): - def test_await_expr(self): - self.validate("""async def foo(): - await x - """) - - self.validate("""async def foo(): - [i async for i in b] - """) - - self.validate("""async def foo(): - {i for i in b - async for i in a if await i - for b in i} - """) - - self.validate("""async def foo(): - [await i for i in b if await c] - """) - - self.validate("""async def foo(): - [ i for i in b if c] - """) - - self.validate("""async def foo(): - - def foo(): pass - - def foo(): pass - - await x - """) - - self.validate("""async def foo(): return await a""") - - self.validate("""def foo(): - def foo(): pass - async def foo(): await x - """) - - self.invalid_syntax("await x") - self.invalid_syntax("""def foo(): - await x""") - - self.invalid_syntax("""def foo(): - def foo(): pass - async def foo(): pass - await x - """) - - def test_async_var(self): - self.validate("""async = 1""") - self.validate("""await = 1""") - self.validate("""def async(): pass""") - - def test_async_for(self): - self.validate("""async def foo(): - async for a in b: pass""") - - def test_async_with(self): - self.validate("""async def foo(): - async with a: pass""") - - self.invalid_syntax("""def foo(): - async with a: pass""") - - def test_async_generator(self): - self.validate( - """async def foo(): - return (i * 2 async for i in arange(42))""" - ) - self.validate( - """def foo(): - return (i * 2 async for i in arange(42))""" - ) - - -class TestRaiseChanges(GrammarTest): - def test_2x_style_1(self): - self.validate("raise") - - def test_2x_style_2(self): - self.validate("raise E, V") - - def test_2x_style_3(self): - self.validate("raise E, V, T") - - def test_2x_style_invalid_1(self): - self.invalid_syntax("raise E, V, T, Z") - - def test_3x_style(self): - self.validate("raise E1 from E2") - - def test_3x_style_invalid_1(self): - self.invalid_syntax("raise E, V from E1") - - def test_3x_style_invalid_2(self): - self.invalid_syntax("raise E from E1, E2") - - def test_3x_style_invalid_3(self): - self.invalid_syntax("raise from E1, E2") - - def test_3x_style_invalid_4(self): - self.invalid_syntax("raise E from") - - -# Modelled after Lib/test/test_grammar.py:TokenTests.test_funcdef issue2292 -# and Lib/test/text_parser.py test_list_displays, test_set_displays, -# test_dict_displays, test_argument_unpacking, ... changes. -class TestUnpackingGeneralizations(GrammarTest): - def test_mid_positional_star(self): - self.validate("""func(1, *(2, 3), 4)""") - - def test_double_star_dict_literal(self): - self.validate("""func(**{'eggs':'scrambled', 'spam':'fried'})""") - - def test_double_star_dict_literal_after_keywords(self): - self.validate("""func(spam='fried', **{'eggs':'scrambled'})""") - - def test_double_star_expression(self): - self.validate("""func(**{'a':2} or {})""") - self.validate("""func(**() or {})""") - - def test_star_expression(self): - self.validate("""func(*[] or [2])""") - - def test_list_display(self): - self.validate("""[*{2}, 3, *[4]]""") - - def test_set_display(self): - self.validate("""{*{2}, 3, *[4]}""") - - def test_dict_display_1(self): - self.validate("""{**{}}""") - - def test_dict_display_2(self): - self.validate("""{**{}, 3:4, **{5:6, 7:8}}""") - - def test_complex_star_expression(self): - self.validate("func(* [] or [1])") - - def test_complex_double_star_expression(self): - self.validate("func(**{1: 3} if False else {x: x for x in range(3)})") - - def test_argument_unpacking_1(self): - self.validate("""f(a, *b, *c, d)""") - - def test_argument_unpacking_2(self): - self.validate("""f(**a, **b)""") - - def test_argument_unpacking_3(self): - self.validate("""f(2, *a, *b, **b, **c, **d)""") - - def test_trailing_commas_1(self): - self.validate("def f(a, b): call(a, b)") - self.validate("def f(a, b,): call(a, b,)") - - def test_trailing_commas_2(self): - self.validate("def f(a, *b): call(a, *b)") - self.validate("def f(a, *b,): call(a, *b,)") - - def test_trailing_commas_3(self): - self.validate("def f(a, b=1): call(a, b=1)") - self.validate("def f(a, b=1,): call(a, b=1,)") - - def test_trailing_commas_4(self): - self.validate("def f(a, **b): call(a, **b)") - self.validate("def f(a, **b,): call(a, **b,)") - - def test_trailing_commas_5(self): - self.validate("def f(*a, b=1): call(*a, b=1)") - self.validate("def f(*a, b=1,): call(*a, b=1,)") - - def test_trailing_commas_6(self): - self.validate("def f(*a, **b): call(*a, **b)") - self.validate("def f(*a, **b,): call(*a, **b,)") - - def test_trailing_commas_7(self): - self.validate("def f(*, b=1): call(*b)") - self.validate("def f(*, b=1,): call(*b,)") - - def test_trailing_commas_8(self): - self.validate("def f(a=1, b=2): call(a=1, b=2)") - self.validate("def f(a=1, b=2,): call(a=1, b=2,)") - - def test_trailing_commas_9(self): - self.validate("def f(a=1, **b): call(a=1, **b)") - self.validate("def f(a=1, **b,): call(a=1, **b,)") - - def test_trailing_commas_lambda_1(self): - self.validate("f = lambda a, b: call(a, b)") - self.validate("f = lambda a, b,: call(a, b,)") - - def test_trailing_commas_lambda_2(self): - self.validate("f = lambda a, *b: call(a, *b)") - self.validate("f = lambda a, *b,: call(a, *b,)") - - def test_trailing_commas_lambda_3(self): - self.validate("f = lambda a, b=1: call(a, b=1)") - self.validate("f = lambda a, b=1,: call(a, b=1,)") - - def test_trailing_commas_lambda_4(self): - self.validate("f = lambda a, **b: call(a, **b)") - self.validate("f = lambda a, **b,: call(a, **b,)") - - def test_trailing_commas_lambda_5(self): - self.validate("f = lambda *a, b=1: call(*a, b=1)") - self.validate("f = lambda *a, b=1,: call(*a, b=1,)") - - def test_trailing_commas_lambda_6(self): - self.validate("f = lambda *a, **b: call(*a, **b)") - self.validate("f = lambda *a, **b,: call(*a, **b,)") - - def test_trailing_commas_lambda_7(self): - self.validate("f = lambda *, b=1: call(*b)") - self.validate("f = lambda *, b=1,: call(*b,)") - - def test_trailing_commas_lambda_8(self): - self.validate("f = lambda a=1, b=2: call(a=1, b=2)") - self.validate("f = lambda a=1, b=2,: call(a=1, b=2,)") - - def test_trailing_commas_lambda_9(self): - self.validate("f = lambda a=1, **b: call(a=1, **b)") - self.validate("f = lambda a=1, **b,: call(a=1, **b,)") - - -# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef -class TestFunctionAnnotations(GrammarTest): - def test_1(self): - self.validate("""def f(x) -> list: pass""") - - def test_2(self): - self.validate("""def f(x:int): pass""") - - def test_3(self): - self.validate("""def f(*x:str): pass""") - - def test_4(self): - self.validate("""def f(**x:float): pass""") - - def test_5(self): - self.validate("""def f(x, y:1+2): pass""") - - def test_6(self): - self.validate("""def f(a, (b:1, c:2, d)): pass""") - - def test_7(self): - self.validate("""def f(a, (b:1, c:2, d), e:3=4, f=5, *g:6): pass""") - - def test_8(self): - s = """def f(a, (b:1, c:2, d), e:3=4, f=5, - *g:6, h:7, i=8, j:9=10, **k:11) -> 12: pass""" - self.validate(s) - - def test_9(self): - s = """def f( - a: str, - b: int, - *, - c: bool = False, - **kwargs, - ) -> None: - call(c=c, **kwargs,)""" - self.validate(s) - - def test_10(self): - s = """def f( - a: str, - ) -> None: - call(a,)""" - self.validate(s) - - def test_11(self): - s = """def f( - a: str = '', - ) -> None: - call(a=a,)""" - self.validate(s) - - def test_12(self): - s = """def f( - *args: str, - ) -> None: - call(*args,)""" - self.validate(s) - - def test_13(self): - self.validate("def f(a: str, b: int) -> None: call(a, b)") - self.validate("def f(a: str, b: int,) -> None: call(a, b,)") - - def test_14(self): - self.validate("def f(a: str, *b: int) -> None: call(a, *b)") - self.validate("def f(a: str, *b: int,) -> None: call(a, *b,)") - - def test_15(self): - self.validate("def f(a: str, b: int=1) -> None: call(a, b=1)") - self.validate("def f(a: str, b: int=1,) -> None: call(a, b=1,)") - - def test_16(self): - self.validate("def f(a: str, **b: int) -> None: call(a, **b)") - self.validate("def f(a: str, **b: int,) -> None: call(a, **b,)") - - def test_17(self): - self.validate("def f(*a: str, b: int=1) -> None: call(*a, b=1)") - self.validate("def f(*a: str, b: int=1,) -> None: call(*a, b=1,)") - - def test_18(self): - self.validate("def f(*a: str, **b: int) -> None: call(*a, **b)") - self.validate("def f(*a: str, **b: int,) -> None: call(*a, **b,)") - - def test_19(self): - self.validate("def f(*, b: int=1) -> None: call(*b)") - self.validate("def f(*, b: int=1,) -> None: call(*b,)") - - def test_20(self): - self.validate("def f(a: str='', b: int=2) -> None: call(a=a, b=2)") - self.validate("def f(a: str='', b: int=2,) -> None: call(a=a, b=2,)") - - def test_21(self): - self.validate("def f(a: str='', **b: int) -> None: call(a=a, **b)") - self.validate("def f(a: str='', **b: int,) -> None: call(a=a, **b,)") - - -# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.test_var_annot -class TestVarAnnotations(GrammarTest): - def test_1(self): - self.validate("var1: int = 5") - - def test_2(self): - self.validate("var2: [int, str]") - - def test_3(self): - self.validate("def f():\n" - " st: str = 'Hello'\n" - " a.b: int = (1, 2)\n" - " return st\n") - - def test_4(self): - self.validate("def fbad():\n" - " x: int\n" - " print(x)\n") - - def test_5(self): - self.validate("class C:\n" - " x: int\n" - " s: str = 'attr'\n" - " z = 2\n" - " def __init__(self, x):\n" - " self.x: int = x\n") - - def test_6(self): - self.validate("lst: List[int] = []") - - -class TestExcept(GrammarTest): - def test_new(self): - s = """ - try: - x - except E as N: - y""" - self.validate(s) - - def test_old(self): - s = """ - try: - x - except E, N: - y""" - self.validate(s) - - -class TestStringLiterals(GrammarTest): - prefixes = ("'", '"', - "r'", 'r"', "R'", 'R"', - "u'", 'u"', "U'", 'U"', - "b'", 'b"', "B'", 'B"', - "f'", 'f"', "F'", 'F"', - "ur'", 'ur"', "Ur'", 'Ur"', - "uR'", 'uR"', "UR'", 'UR"', - "br'", 'br"', "Br'", 'Br"', - "bR'", 'bR"', "BR'", 'BR"', - "rb'", 'rb"', "Rb'", 'Rb"', - "rB'", 'rB"', "RB'", 'RB"',) - - def test_lit(self): - for pre in self.prefixes: - single = "{p}spamspamspam{s}".format(p=pre, s=pre[-1]) - self.validate(single) - triple = "{p}{s}{s}eggs{s}{s}{s}".format(p=pre, s=pre[-1]) - self.validate(triple) - - -# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testAtoms -class TestSetLiteral(GrammarTest): - def test_1(self): - self.validate("""x = {'one'}""") - - def test_2(self): - self.validate("""x = {'one', 1,}""") - - def test_3(self): - self.validate("""x = {'one', 'two', 'three'}""") - - def test_4(self): - self.validate("""x = {2, 3, 4,}""") - - -# Adapted from Python 3's Lib/test/test_unicode_identifiers.py and -# Lib/test/test_tokenize.py:TokenizeTest.test_non_ascii_identifiers -class TestIdentifier(GrammarTest): - def test_non_ascii_identifiers(self): - self.validate("Örter = 'places'\ngrün = 'green'") - self.validate("蟒 = a蟒 = 锦蛇 = 1") - self.validate("µ = aµ = µµ = 1") - self.validate("𝔘𝔫𝔦𝔠𝔬𝔡𝔢 = a_𝔘𝔫𝔦𝔠𝔬𝔡𝔢 = 1") - - -class TestNumericLiterals(GrammarTest): - def test_new_octal_notation(self): - self.validate("""0o7777777777777""") - self.invalid_syntax("""0o7324528887""") - - def test_new_binary_notation(self): - self.validate("""0b101010""") - self.invalid_syntax("""0b0101021""") - - -class TestClassDef(GrammarTest): - def test_new_syntax(self): - self.validate("class B(t=7): pass") - self.validate("class B(t, *args): pass") - self.validate("class B(t, **kwargs): pass") - self.validate("class B(t, *args, **kwargs): pass") - self.validate("class B(t, y=9, *args, **kwargs,): pass") - - -class TestParserIdempotency(support.TestCase): - - """A cut-down version of pytree_idempotency.py.""" - - def parse_file(self, filepath): - if test.support.verbose: - print(f"Parse file: {filepath}") - with open(filepath, "rb") as fp: - encoding = tokenize.detect_encoding(fp.readline)[0] - self.assertIsNotNone(encoding, - "can't detect encoding for %s" % filepath) - with open(filepath, "r", encoding=encoding) as fp: - source = fp.read() - try: - tree = driver.parse_string(source) - except ParseError: - try: - tree = driver_no_print_statement.parse_string(source) - except ParseError as err: - self.fail('ParseError on file %s (%s)' % (filepath, err)) - new = str(tree) - if new != source: - print(diff_texts(source, new, filepath)) - self.fail("Idempotency failed: %s" % filepath) - - def test_all_project_files(self): - for filepath in support.all_project_files(): - with self.subTest(filepath=filepath): - self.parse_file(filepath) - - def test_extended_unpacking(self): - driver.parse_string("a, *b, c = x\n") - driver.parse_string("[*a, b] = x\n") - driver.parse_string("(z, *y, w) = m\n") - driver.parse_string("for *z, m in d: pass\n") - - -class TestLiterals(GrammarTest): - - def validate(self, s): - driver.parse_string(support.dedent(s) + "\n\n") - - def test_multiline_bytes_literals(self): - s = """ - md5test(b"\xaa" * 80, - (b"Test Using Larger Than Block-Size Key " - b"and Larger Than One Block-Size Data"), - "6f630fad67cda0ee1fb1f562db3aa53e") - """ - self.validate(s) - - def test_multiline_bytes_tripquote_literals(self): - s = ''' - b""" - - - """ - ''' - self.validate(s) - - def test_multiline_str_literals(self): - s = """ - md5test("\xaa" * 80, - ("Test Using Larger Than Block-Size Key " - "and Larger Than One Block-Size Data"), - "6f630fad67cda0ee1fb1f562db3aa53e") - """ - self.validate(s) - - -class TestNamedAssignments(GrammarTest): - """Also known as the walrus operator.""" - - def test_named_assignment_if(self): - driver.parse_string("if f := x(): pass\n") - - def test_named_assignment_while(self): - driver.parse_string("while f := x(): pass\n") - - def test_named_assignment_generator(self): - driver.parse_string("any((lastNum := num) == 1 for num in [1, 2, 3])\n") - - def test_named_assignment_listcomp(self): - driver.parse_string("[(lastNum := num) == 1 for num in [1, 2, 3]]\n") - - -class TestPositionalOnlyArgs(GrammarTest): - - def test_one_pos_only_arg(self): - driver.parse_string("def one_pos_only_arg(a, /): pass\n") - - def test_all_markers(self): - driver.parse_string( - "def all_markers(a, b=2, /, c, d=4, *, e=5, f): pass\n") - - def test_all_with_args_and_kwargs(self): - driver.parse_string( - """def all_markers_with_args_and_kwargs( - aa, b, /, _cc, d, *args, e, f_f, **kwargs, - ): - pass\n""") - - def test_lambda_soup(self): - driver.parse_string( - "lambda a, b, /, c, d, *args, e, f, **kw: kw\n") - - def test_only_positional_or_keyword(self): - driver.parse_string("def func(a,b,/,*,g,e=3): pass\n") - - -class TestPickleableException(unittest.TestCase): - def test_ParseError(self): - err = ParseError('msg', 2, None, (1, 'context')) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - err2 = pickle.loads(pickle.dumps(err, protocol=proto)) - self.assertEqual(err.args, err2.args) - self.assertEqual(err.msg, err2.msg) - self.assertEqual(err.type, err2.type) - self.assertEqual(err.value, err2.value) - self.assertEqual(err.context, err2.context) - - -def diff_texts(a, b, filename): - a = a.splitlines() - b = b.splitlines() - return difflib.unified_diff(a, b, filename, filename, - "(original)", "(reserialized)", - lineterm="") - - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/test/test_lib2to3/test_pytree.py b/Lib/test/test_lib2to3/test_pytree.py deleted file mode 100644 index 177126d5452910..00000000000000 --- a/Lib/test/test_lib2to3/test_pytree.py +++ /dev/null @@ -1,472 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Unit tests for pytree.py. - -NOTE: Please *don't* add doc strings to individual test methods! -In verbose mode, printing of the module, class and method name is much -more helpful than printing of (the first line of) the docstring, -especially when debugging a test. -""" - -# Testing imports -from . import support - -from lib2to3 import pytree - -try: - sorted -except NameError: - def sorted(lst): - l = list(lst) - l.sort() - return l - -class TestNodes(support.TestCase): - - """Unit tests for nodes (Base, Leaf, Node).""" - - def test_instantiate_base(self): - if __debug__: - # Test that instantiating Base() raises an AssertionError - self.assertRaises(AssertionError, pytree.Base) - - def test_leaf(self): - l1 = pytree.Leaf(100, "foo") - self.assertEqual(l1.type, 100) - self.assertEqual(l1.value, "foo") - - def test_leaf_repr(self): - l1 = pytree.Leaf(100, "foo") - self.assertEqual(repr(l1), "Leaf(100, 'foo')") - - def test_leaf_str(self): - l1 = pytree.Leaf(100, "foo") - self.assertEqual(str(l1), "foo") - l2 = pytree.Leaf(100, "foo", context=(" ", (10, 1))) - self.assertEqual(str(l2), " foo") - - def test_leaf_str_numeric_value(self): - # Make sure that the Leaf's value is stringified. Failing to - # do this can cause a TypeError in certain situations. - l1 = pytree.Leaf(2, 5) - l1.prefix = "foo_" - self.assertEqual(str(l1), "foo_5") - - def test_leaf_equality(self): - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(100, "foo", context=(" ", (1, 0))) - self.assertEqual(l1, l2) - l3 = pytree.Leaf(101, "foo") - l4 = pytree.Leaf(100, "bar") - self.assertNotEqual(l1, l3) - self.assertNotEqual(l1, l4) - - def test_leaf_prefix(self): - l1 = pytree.Leaf(100, "foo") - self.assertEqual(l1.prefix, "") - self.assertFalse(l1.was_changed) - l1.prefix = " ##\n\n" - self.assertEqual(l1.prefix, " ##\n\n") - self.assertTrue(l1.was_changed) - - def test_node(self): - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(200, "bar") - n1 = pytree.Node(1000, [l1, l2]) - self.assertEqual(n1.type, 1000) - self.assertEqual(n1.children, [l1, l2]) - - def test_node_repr(self): - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(100, "bar", context=(" ", (1, 0))) - n1 = pytree.Node(1000, [l1, l2]) - self.assertEqual(repr(n1), - "Node(1000, [%s, %s])" % (repr(l1), repr(l2))) - - def test_node_str(self): - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(100, "bar", context=(" ", (1, 0))) - n1 = pytree.Node(1000, [l1, l2]) - self.assertEqual(str(n1), "foo bar") - - def test_node_prefix(self): - l1 = pytree.Leaf(100, "foo") - self.assertEqual(l1.prefix, "") - n1 = pytree.Node(1000, [l1]) - self.assertEqual(n1.prefix, "") - n1.prefix = " " - self.assertEqual(n1.prefix, " ") - self.assertEqual(l1.prefix, " ") - - def test_get_suffix(self): - l1 = pytree.Leaf(100, "foo", prefix="a") - l2 = pytree.Leaf(100, "bar", prefix="b") - n1 = pytree.Node(1000, [l1, l2]) - - self.assertEqual(l1.get_suffix(), l2.prefix) - self.assertEqual(l2.get_suffix(), "") - self.assertEqual(n1.get_suffix(), "") - - l3 = pytree.Leaf(100, "bar", prefix="c") - n2 = pytree.Node(1000, [n1, l3]) - - self.assertEqual(n1.get_suffix(), l3.prefix) - self.assertEqual(l3.get_suffix(), "") - self.assertEqual(n2.get_suffix(), "") - - def test_node_equality(self): - n1 = pytree.Node(1000, ()) - n2 = pytree.Node(1000, [], context=(" ", (1, 0))) - self.assertEqual(n1, n2) - n3 = pytree.Node(1001, ()) - self.assertNotEqual(n1, n3) - - def test_node_recursive_equality(self): - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(100, "foo") - n1 = pytree.Node(1000, [l1]) - n2 = pytree.Node(1000, [l2]) - self.assertEqual(n1, n2) - l3 = pytree.Leaf(100, "bar") - n3 = pytree.Node(1000, [l3]) - self.assertNotEqual(n1, n3) - - def test_replace(self): - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(100, "+") - l3 = pytree.Leaf(100, "bar") - n1 = pytree.Node(1000, [l1, l2, l3]) - self.assertEqual(n1.children, [l1, l2, l3]) - self.assertIsInstance(n1.children, list) - self.assertFalse(n1.was_changed) - l2new = pytree.Leaf(100, "-") - l2.replace(l2new) - self.assertEqual(n1.children, [l1, l2new, l3]) - self.assertIsInstance(n1.children, list) - self.assertTrue(n1.was_changed) - - def test_replace_with_list(self): - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(100, "+") - l3 = pytree.Leaf(100, "bar") - n1 = pytree.Node(1000, [l1, l2, l3]) - - l2.replace([pytree.Leaf(100, "*"), pytree.Leaf(100, "*")]) - self.assertEqual(str(n1), "foo**bar") - self.assertIsInstance(n1.children, list) - - def test_leaves(self): - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(100, "bar") - l3 = pytree.Leaf(100, "fooey") - n2 = pytree.Node(1000, [l1, l2]) - n3 = pytree.Node(1000, [l3]) - n1 = pytree.Node(1000, [n2, n3]) - - self.assertEqual(list(n1.leaves()), [l1, l2, l3]) - - def test_depth(self): - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(100, "bar") - n2 = pytree.Node(1000, [l1, l2]) - n3 = pytree.Node(1000, []) - n1 = pytree.Node(1000, [n2, n3]) - - self.assertEqual(l1.depth(), 2) - self.assertEqual(n3.depth(), 1) - self.assertEqual(n1.depth(), 0) - - def test_post_order(self): - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(100, "bar") - l3 = pytree.Leaf(100, "fooey") - c1 = pytree.Node(1000, [l1, l2]) - n1 = pytree.Node(1000, [c1, l3]) - self.assertEqual(list(n1.post_order()), [l1, l2, c1, l3, n1]) - - def test_pre_order(self): - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(100, "bar") - l3 = pytree.Leaf(100, "fooey") - c1 = pytree.Node(1000, [l1, l2]) - n1 = pytree.Node(1000, [c1, l3]) - self.assertEqual(list(n1.pre_order()), [n1, c1, l1, l2, l3]) - - def test_changed(self): - l1 = pytree.Leaf(100, "f") - self.assertFalse(l1.was_changed) - l1.changed() - self.assertTrue(l1.was_changed) - - l1 = pytree.Leaf(100, "f") - n1 = pytree.Node(1000, [l1]) - self.assertFalse(n1.was_changed) - n1.changed() - self.assertTrue(n1.was_changed) - - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(100, "+") - l3 = pytree.Leaf(100, "bar") - n1 = pytree.Node(1000, [l1, l2, l3]) - n2 = pytree.Node(1000, [n1]) - self.assertFalse(l1.was_changed) - self.assertFalse(n1.was_changed) - self.assertFalse(n2.was_changed) - - n1.changed() - self.assertTrue(n1.was_changed) - self.assertTrue(n2.was_changed) - self.assertFalse(l1.was_changed) - - def test_leaf_constructor_prefix(self): - for prefix in ("xyz_", ""): - l1 = pytree.Leaf(100, "self", prefix=prefix) - self.assertTrue(str(l1), prefix + "self") - self.assertEqual(l1.prefix, prefix) - - def test_node_constructor_prefix(self): - for prefix in ("xyz_", ""): - l1 = pytree.Leaf(100, "self") - l2 = pytree.Leaf(100, "foo", prefix="_") - n1 = pytree.Node(1000, [l1, l2], prefix=prefix) - self.assertTrue(str(n1), prefix + "self_foo") - self.assertEqual(n1.prefix, prefix) - self.assertEqual(l1.prefix, prefix) - self.assertEqual(l2.prefix, "_") - - def test_remove(self): - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(100, "foo") - n1 = pytree.Node(1000, [l1, l2]) - n2 = pytree.Node(1000, [n1]) - - self.assertEqual(n1.remove(), 0) - self.assertEqual(n2.children, []) - self.assertEqual(l1.parent, n1) - self.assertEqual(n1.parent, None) - self.assertEqual(n2.parent, None) - self.assertFalse(n1.was_changed) - self.assertTrue(n2.was_changed) - - self.assertEqual(l2.remove(), 1) - self.assertEqual(l1.remove(), 0) - self.assertEqual(n1.children, []) - self.assertEqual(l1.parent, None) - self.assertEqual(n1.parent, None) - self.assertEqual(n2.parent, None) - self.assertTrue(n1.was_changed) - self.assertTrue(n2.was_changed) - - def test_remove_parentless(self): - n1 = pytree.Node(1000, []) - n1.remove() - self.assertEqual(n1.parent, None) - - l1 = pytree.Leaf(100, "foo") - l1.remove() - self.assertEqual(l1.parent, None) - - def test_node_set_child(self): - l1 = pytree.Leaf(100, "foo") - n1 = pytree.Node(1000, [l1]) - - l2 = pytree.Leaf(100, "bar") - n1.set_child(0, l2) - self.assertEqual(l1.parent, None) - self.assertEqual(l2.parent, n1) - self.assertEqual(n1.children, [l2]) - - n2 = pytree.Node(1000, [l1]) - n2.set_child(0, n1) - self.assertEqual(l1.parent, None) - self.assertEqual(n1.parent, n2) - self.assertEqual(n2.parent, None) - self.assertEqual(n2.children, [n1]) - - self.assertRaises(IndexError, n1.set_child, 4, l2) - # I don't care what it raises, so long as it's an exception - self.assertRaises(Exception, n1.set_child, 0, list) - - def test_node_insert_child(self): - l1 = pytree.Leaf(100, "foo") - n1 = pytree.Node(1000, [l1]) - - l2 = pytree.Leaf(100, "bar") - n1.insert_child(0, l2) - self.assertEqual(l2.parent, n1) - self.assertEqual(n1.children, [l2, l1]) - - l3 = pytree.Leaf(100, "abc") - n1.insert_child(2, l3) - self.assertEqual(n1.children, [l2, l1, l3]) - - # I don't care what it raises, so long as it's an exception - self.assertRaises(Exception, n1.insert_child, 0, list) - - def test_node_append_child(self): - n1 = pytree.Node(1000, []) - - l1 = pytree.Leaf(100, "foo") - n1.append_child(l1) - self.assertEqual(l1.parent, n1) - self.assertEqual(n1.children, [l1]) - - l2 = pytree.Leaf(100, "bar") - n1.append_child(l2) - self.assertEqual(l2.parent, n1) - self.assertEqual(n1.children, [l1, l2]) - - # I don't care what it raises, so long as it's an exception - self.assertRaises(Exception, n1.append_child, list) - - def test_node_next_sibling(self): - n1 = pytree.Node(1000, []) - n2 = pytree.Node(1000, []) - p1 = pytree.Node(1000, [n1, n2]) - - self.assertIs(n1.next_sibling, n2) - self.assertEqual(n2.next_sibling, None) - self.assertEqual(p1.next_sibling, None) - - def test_leaf_next_sibling(self): - l1 = pytree.Leaf(100, "a") - l2 = pytree.Leaf(100, "b") - p1 = pytree.Node(1000, [l1, l2]) - - self.assertIs(l1.next_sibling, l2) - self.assertEqual(l2.next_sibling, None) - self.assertEqual(p1.next_sibling, None) - - def test_node_prev_sibling(self): - n1 = pytree.Node(1000, []) - n2 = pytree.Node(1000, []) - p1 = pytree.Node(1000, [n1, n2]) - - self.assertIs(n2.prev_sibling, n1) - self.assertEqual(n1.prev_sibling, None) - self.assertEqual(p1.prev_sibling, None) - - def test_leaf_prev_sibling(self): - l1 = pytree.Leaf(100, "a") - l2 = pytree.Leaf(100, "b") - p1 = pytree.Node(1000, [l1, l2]) - - self.assertIs(l2.prev_sibling, l1) - self.assertEqual(l1.prev_sibling, None) - self.assertEqual(p1.prev_sibling, None) - - -class TestPatterns(support.TestCase): - - """Unit tests for tree matching patterns.""" - - def test_basic_patterns(self): - # Build a tree - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(100, "bar") - l3 = pytree.Leaf(100, "foo") - n1 = pytree.Node(1000, [l1, l2]) - n2 = pytree.Node(1000, [l3]) - root = pytree.Node(1000, [n1, n2]) - # Build a pattern matching a leaf - pl = pytree.LeafPattern(100, "foo", name="pl") - r = {} - self.assertFalse(pl.match(root, results=r)) - self.assertEqual(r, {}) - self.assertFalse(pl.match(n1, results=r)) - self.assertEqual(r, {}) - self.assertFalse(pl.match(n2, results=r)) - self.assertEqual(r, {}) - self.assertTrue(pl.match(l1, results=r)) - self.assertEqual(r, {"pl": l1}) - r = {} - self.assertFalse(pl.match(l2, results=r)) - self.assertEqual(r, {}) - # Build a pattern matching a node - pn = pytree.NodePattern(1000, [pl], name="pn") - self.assertFalse(pn.match(root, results=r)) - self.assertEqual(r, {}) - self.assertFalse(pn.match(n1, results=r)) - self.assertEqual(r, {}) - self.assertTrue(pn.match(n2, results=r)) - self.assertEqual(r, {"pn": n2, "pl": l3}) - r = {} - self.assertFalse(pn.match(l1, results=r)) - self.assertEqual(r, {}) - self.assertFalse(pn.match(l2, results=r)) - self.assertEqual(r, {}) - - def test_wildcard(self): - # Build a tree for testing - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(100, "bar") - l3 = pytree.Leaf(100, "foo") - n1 = pytree.Node(1000, [l1, l2]) - n2 = pytree.Node(1000, [l3]) - root = pytree.Node(1000, [n1, n2]) - # Build a pattern - pl = pytree.LeafPattern(100, "foo", name="pl") - pn = pytree.NodePattern(1000, [pl], name="pn") - pw = pytree.WildcardPattern([[pn], [pl, pl]], name="pw") - r = {} - self.assertFalse(pw.match_seq([root], r)) - self.assertEqual(r, {}) - self.assertFalse(pw.match_seq([n1], r)) - self.assertEqual(r, {}) - self.assertTrue(pw.match_seq([n2], r)) - # These are easier to debug - self.assertEqual(sorted(r.keys()), ["pl", "pn", "pw"]) - self.assertEqual(r["pl"], l1) - self.assertEqual(r["pn"], n2) - self.assertEqual(r["pw"], [n2]) - # But this is equivalent - self.assertEqual(r, {"pl": l1, "pn": n2, "pw": [n2]}) - r = {} - self.assertTrue(pw.match_seq([l1, l3], r)) - self.assertEqual(r, {"pl": l3, "pw": [l1, l3]}) - self.assertIs(r["pl"], l3) - r = {} - - def test_generate_matches(self): - la = pytree.Leaf(1, "a") - lb = pytree.Leaf(1, "b") - lc = pytree.Leaf(1, "c") - ld = pytree.Leaf(1, "d") - le = pytree.Leaf(1, "e") - lf = pytree.Leaf(1, "f") - leaves = [la, lb, lc, ld, le, lf] - root = pytree.Node(1000, leaves) - pa = pytree.LeafPattern(1, "a", "pa") - pb = pytree.LeafPattern(1, "b", "pb") - pc = pytree.LeafPattern(1, "c", "pc") - pd = pytree.LeafPattern(1, "d", "pd") - pe = pytree.LeafPattern(1, "e", "pe") - pf = pytree.LeafPattern(1, "f", "pf") - pw = pytree.WildcardPattern([[pa, pb, pc], [pd, pe], - [pa, pb], [pc, pd], [pe, pf]], - min=1, max=4, name="pw") - self.assertEqual([x[0] for x in pw.generate_matches(leaves)], - [3, 5, 2, 4, 6]) - pr = pytree.NodePattern(type=1000, content=[pw], name="pr") - matches = list(pytree.generate_matches([pr], [root])) - self.assertEqual(len(matches), 1) - c, r = matches[0] - self.assertEqual(c, 1) - self.assertEqual(str(r["pr"]), "abcdef") - self.assertEqual(r["pw"], [la, lb, lc, ld, le, lf]) - for c in "abcdef": - self.assertEqual(r["p" + c], pytree.Leaf(1, c)) - - def test_has_key_example(self): - pattern = pytree.NodePattern(331, - (pytree.LeafPattern(7), - pytree.WildcardPattern(name="args"), - pytree.LeafPattern(8))) - l1 = pytree.Leaf(7, "(") - l2 = pytree.Leaf(3, "x") - l3 = pytree.Leaf(8, ")") - node = pytree.Node(331, [l1, l2, l3]) - r = {} - self.assertTrue(pattern.match(node, r)) - self.assertEqual(r["args"], [l2]) diff --git a/Lib/test/test_lib2to3/test_refactor.py b/Lib/test/test_lib2to3/test_refactor.py deleted file mode 100644 index be705679f06dba..00000000000000 --- a/Lib/test/test_lib2to3/test_refactor.py +++ /dev/null @@ -1,337 +0,0 @@ -""" -Unit tests for refactor.py. -""" - -import sys -import os -import codecs -import io -import re -import tempfile -import shutil -import unittest - -from lib2to3 import refactor, pygram, fixer_base -from lib2to3.pgen2 import token - - -TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "data") -FIXER_DIR = os.path.join(TEST_DATA_DIR, "fixers") - -sys.path.append(FIXER_DIR) -try: - _DEFAULT_FIXERS = refactor.get_fixers_from_package("myfixes") -finally: - sys.path.pop() - -_2TO3_FIXERS = refactor.get_fixers_from_package("lib2to3.fixes") - -class TestRefactoringTool(unittest.TestCase): - - def setUp(self): - sys.path.append(FIXER_DIR) - - def tearDown(self): - sys.path.pop() - - def check_instances(self, instances, classes): - for inst, cls in zip(instances, classes): - if not isinstance(inst, cls): - self.fail("%s are not instances of %s" % instances, classes) - - def rt(self, options=None, fixers=_DEFAULT_FIXERS, explicit=None): - return refactor.RefactoringTool(fixers, options, explicit) - - def test_print_function_option(self): - rt = self.rt({"print_function" : True}) - self.assertNotIn("print", rt.grammar.keywords) - self.assertNotIn("print", rt.driver.grammar.keywords) - - def test_exec_function_option(self): - rt = self.rt({"exec_function" : True}) - self.assertNotIn("exec", rt.grammar.keywords) - self.assertNotIn("exec", rt.driver.grammar.keywords) - - def test_write_unchanged_files_option(self): - rt = self.rt() - self.assertFalse(rt.write_unchanged_files) - rt = self.rt({"write_unchanged_files" : True}) - self.assertTrue(rt.write_unchanged_files) - - def test_fixer_loading_helpers(self): - contents = ["explicit", "first", "last", "parrot", "preorder"] - non_prefixed = refactor.get_all_fix_names("myfixes") - prefixed = refactor.get_all_fix_names("myfixes", False) - full_names = refactor.get_fixers_from_package("myfixes") - self.assertEqual(prefixed, ["fix_" + name for name in contents]) - self.assertEqual(non_prefixed, contents) - self.assertEqual(full_names, - ["myfixes.fix_" + name for name in contents]) - - def test_detect_future_features(self): - run = refactor._detect_future_features - fs = frozenset - empty = fs() - self.assertEqual(run(""), empty) - self.assertEqual(run("from __future__ import print_function"), - fs(("print_function",))) - self.assertEqual(run("from __future__ import generators"), - fs(("generators",))) - self.assertEqual(run("from __future__ import generators, feature"), - fs(("generators", "feature"))) - inp = "from __future__ import generators, print_function" - self.assertEqual(run(inp), fs(("generators", "print_function"))) - inp ="from __future__ import print_function, generators" - self.assertEqual(run(inp), fs(("print_function", "generators"))) - inp = "from __future__ import (print_function,)" - self.assertEqual(run(inp), fs(("print_function",))) - inp = "from __future__ import (generators, print_function)" - self.assertEqual(run(inp), fs(("generators", "print_function"))) - inp = "from __future__ import (generators, nested_scopes)" - self.assertEqual(run(inp), fs(("generators", "nested_scopes"))) - inp = """from __future__ import generators -from __future__ import print_function""" - self.assertEqual(run(inp), fs(("generators", "print_function"))) - invalid = ("from", - "from 4", - "from x", - "from x 5", - "from x im", - "from x import", - "from x import 4", - ) - for inp in invalid: - self.assertEqual(run(inp), empty) - inp = "'docstring'\nfrom __future__ import print_function" - self.assertEqual(run(inp), fs(("print_function",))) - inp = "'docstring'\n'somng'\nfrom __future__ import print_function" - self.assertEqual(run(inp), empty) - inp = "# comment\nfrom __future__ import print_function" - self.assertEqual(run(inp), fs(("print_function",))) - inp = "# comment\n'doc'\nfrom __future__ import print_function" - self.assertEqual(run(inp), fs(("print_function",))) - inp = "class x: pass\nfrom __future__ import print_function" - self.assertEqual(run(inp), empty) - - def test_get_headnode_dict(self): - class NoneFix(fixer_base.BaseFix): - pass - - class FileInputFix(fixer_base.BaseFix): - PATTERN = "file_input< any * >" - - class SimpleFix(fixer_base.BaseFix): - PATTERN = "'name'" - - no_head = NoneFix({}, []) - with_head = FileInputFix({}, []) - simple = SimpleFix({}, []) - d = refactor._get_headnode_dict([no_head, with_head, simple]) - top_fixes = d.pop(pygram.python_symbols.file_input) - self.assertEqual(top_fixes, [with_head, no_head]) - name_fixes = d.pop(token.NAME) - self.assertEqual(name_fixes, [simple, no_head]) - for fixes in d.values(): - self.assertEqual(fixes, [no_head]) - - def test_fixer_loading(self): - from myfixes.fix_first import FixFirst - from myfixes.fix_last import FixLast - from myfixes.fix_parrot import FixParrot - from myfixes.fix_preorder import FixPreorder - - rt = self.rt() - pre, post = rt.get_fixers() - - self.check_instances(pre, [FixPreorder]) - self.check_instances(post, [FixFirst, FixParrot, FixLast]) - - def test_naughty_fixers(self): - self.assertRaises(ImportError, self.rt, fixers=["not_here"]) - self.assertRaises(refactor.FixerError, self.rt, fixers=["no_fixer_cls"]) - self.assertRaises(refactor.FixerError, self.rt, fixers=["bad_order"]) - - def test_refactor_string(self): - rt = self.rt() - input = "def parrot(): pass\n\n" - tree = rt.refactor_string(input, "") - self.assertNotEqual(str(tree), input) - - input = "def f(): pass\n\n" - tree = rt.refactor_string(input, "") - self.assertEqual(str(tree), input) - - def test_refactor_stdin(self): - - class MyRT(refactor.RefactoringTool): - - def print_output(self, old_text, new_text, filename, equal): - results.extend([old_text, new_text, filename, equal]) - - results = [] - rt = MyRT(_DEFAULT_FIXERS) - save = sys.stdin - sys.stdin = io.StringIO("def parrot(): pass\n\n") - try: - rt.refactor_stdin() - finally: - sys.stdin = save - expected = ["def parrot(): pass\n\n", - "def cheese(): pass\n\n", - "", False] - self.assertEqual(results, expected) - - def check_file_refactoring(self, test_file, fixers=_2TO3_FIXERS, - options=None, mock_log_debug=None, - actually_write=True): - test_file = self.init_test_file(test_file) - old_contents = self.read_file(test_file) - rt = self.rt(fixers=fixers, options=options) - if mock_log_debug: - rt.log_debug = mock_log_debug - - rt.refactor_file(test_file) - self.assertEqual(old_contents, self.read_file(test_file)) - - if not actually_write: - return - rt.refactor_file(test_file, True) - new_contents = self.read_file(test_file) - self.assertNotEqual(old_contents, new_contents) - return new_contents - - def init_test_file(self, test_file): - tmpdir = tempfile.mkdtemp(prefix="2to3-test_refactor") - self.addCleanup(shutil.rmtree, tmpdir) - shutil.copy(test_file, tmpdir) - test_file = os.path.join(tmpdir, os.path.basename(test_file)) - os.chmod(test_file, 0o644) - return test_file - - def read_file(self, test_file): - with open(test_file, "rb") as fp: - return fp.read() - - def refactor_file(self, test_file, fixers=_2TO3_FIXERS): - test_file = self.init_test_file(test_file) - old_contents = self.read_file(test_file) - rt = self.rt(fixers=fixers) - rt.refactor_file(test_file, True) - new_contents = self.read_file(test_file) - return old_contents, new_contents - - def test_refactor_file(self): - test_file = os.path.join(FIXER_DIR, "parrot_example.py") - self.check_file_refactoring(test_file, _DEFAULT_FIXERS) - - def test_refactor_file_write_unchanged_file(self): - test_file = os.path.join(FIXER_DIR, "parrot_example.py") - debug_messages = [] - def recording_log_debug(msg, *args): - debug_messages.append(msg % args) - self.check_file_refactoring(test_file, fixers=(), - options={"write_unchanged_files": True}, - mock_log_debug=recording_log_debug, - actually_write=False) - # Testing that it logged this message when write=False was passed is - # sufficient to see that it did not bail early after "No changes". - message_regex = r"Not writing changes to .*%s" % \ - re.escape(os.sep + os.path.basename(test_file)) - for message in debug_messages: - if "Not writing changes" in message: - self.assertRegex(message, message_regex) - break - else: - self.fail("%r not matched in %r" % (message_regex, debug_messages)) - - def test_refactor_dir(self): - def check(structure, expected): - def mock_refactor_file(self, f, *args): - got.append(f) - save_func = refactor.RefactoringTool.refactor_file - refactor.RefactoringTool.refactor_file = mock_refactor_file - rt = self.rt() - got = [] - dir = tempfile.mkdtemp(prefix="2to3-test_refactor") - try: - os.mkdir(os.path.join(dir, "a_dir")) - for fn in structure: - open(os.path.join(dir, fn), "wb").close() - rt.refactor_dir(dir) - finally: - refactor.RefactoringTool.refactor_file = save_func - shutil.rmtree(dir) - self.assertEqual(got, - [os.path.join(dir, path) for path in expected]) - check([], []) - tree = ["nothing", - "hi.py", - ".dumb", - ".after.py", - "notpy.npy", - "sappy"] - expected = ["hi.py"] - check(tree, expected) - tree = ["hi.py", - os.path.join("a_dir", "stuff.py")] - check(tree, tree) - - def test_file_encoding(self): - fn = os.path.join(TEST_DATA_DIR, "different_encoding.py") - self.check_file_refactoring(fn) - - def test_false_file_encoding(self): - fn = os.path.join(TEST_DATA_DIR, "false_encoding.py") - data = self.check_file_refactoring(fn) - - def test_bom(self): - fn = os.path.join(TEST_DATA_DIR, "bom.py") - data = self.check_file_refactoring(fn) - self.assertTrue(data.startswith(codecs.BOM_UTF8)) - - def test_crlf_newlines(self): - old_sep = os.linesep - os.linesep = "\r\n" - try: - fn = os.path.join(TEST_DATA_DIR, "crlf.py") - fixes = refactor.get_fixers_from_package("lib2to3.fixes") - self.check_file_refactoring(fn, fixes) - finally: - os.linesep = old_sep - - def test_crlf_unchanged(self): - fn = os.path.join(TEST_DATA_DIR, "crlf.py") - old, new = self.refactor_file(fn) - self.assertIn(b"\r\n", old) - self.assertIn(b"\r\n", new) - self.assertNotIn(b"\r\r\n", new) - - def test_refactor_docstring(self): - rt = self.rt() - - doc = """ ->>> example() -42 -""" - out = rt.refactor_docstring(doc, "") - self.assertEqual(out, doc) - - doc = """ ->>> def parrot(): -... return 43 -""" - out = rt.refactor_docstring(doc, "") - self.assertNotEqual(out, doc) - - def test_explicit(self): - from myfixes.fix_explicit import FixExplicit - - rt = self.rt(fixers=["myfixes.fix_explicit"]) - self.assertEqual(len(rt.post_order), 0) - - rt = self.rt(explicit=["myfixes.fix_explicit"]) - for fix in rt.post_order: - if isinstance(fix, FixExplicit): - break - else: - self.fail("explicit fixer not loaded") diff --git a/Lib/test/test_lib2to3/test_util.py b/Lib/test/test_lib2to3/test_util.py deleted file mode 100644 index c6c613972dac7e..00000000000000 --- a/Lib/test/test_lib2to3/test_util.py +++ /dev/null @@ -1,591 +0,0 @@ -""" Test suite for the code in fixer_util """ - -# Testing imports -from . import support - -# Local imports -from lib2to3.pytree import Node, Leaf -from lib2to3 import fixer_util -from lib2to3.fixer_util import Attr, Name, Call, Comma -from lib2to3.pgen2 import token - -def parse(code, strip_levels=0): - # The topmost node is file_input, which we don't care about. - # The next-topmost node is a *_stmt node, which we also don't care about - tree = support.parse_string(code) - for i in range(strip_levels): - tree = tree.children[0] - tree.parent = None - return tree - -class MacroTestCase(support.TestCase): - def assertStr(self, node, string): - if isinstance(node, (tuple, list)): - node = Node(fixer_util.syms.simple_stmt, node) - self.assertEqual(str(node), string) - - -class Test_is_tuple(support.TestCase): - def is_tuple(self, string): - return fixer_util.is_tuple(parse(string, strip_levels=2)) - - def test_valid(self): - self.assertTrue(self.is_tuple("(a, b)")) - self.assertTrue(self.is_tuple("(a, (b, c))")) - self.assertTrue(self.is_tuple("((a, (b, c)),)")) - self.assertTrue(self.is_tuple("(a,)")) - self.assertTrue(self.is_tuple("()")) - - def test_invalid(self): - self.assertFalse(self.is_tuple("(a)")) - self.assertFalse(self.is_tuple("('foo') % (b, c)")) - - -class Test_is_list(support.TestCase): - def is_list(self, string): - return fixer_util.is_list(parse(string, strip_levels=2)) - - def test_valid(self): - self.assertTrue(self.is_list("[]")) - self.assertTrue(self.is_list("[a]")) - self.assertTrue(self.is_list("[a, b]")) - self.assertTrue(self.is_list("[a, [b, c]]")) - self.assertTrue(self.is_list("[[a, [b, c]],]")) - - def test_invalid(self): - self.assertFalse(self.is_list("[]+[]")) - - -class Test_Attr(MacroTestCase): - def test(self): - call = parse("foo()", strip_levels=2) - - self.assertStr(Attr(Name("a"), Name("b")), "a.b") - self.assertStr(Attr(call, Name("b")), "foo().b") - - def test_returns(self): - attr = Attr(Name("a"), Name("b")) - self.assertEqual(type(attr), list) - - -class Test_Name(MacroTestCase): - def test(self): - self.assertStr(Name("a"), "a") - self.assertStr(Name("foo.foo().bar"), "foo.foo().bar") - self.assertStr(Name("a", prefix="b"), "ba") - - -class Test_Call(MacroTestCase): - def _Call(self, name, args=None, prefix=None): - """Help the next test""" - children = [] - if isinstance(args, list): - for arg in args: - children.append(arg) - children.append(Comma()) - children.pop() - return Call(Name(name), children, prefix) - - def test(self): - kids = [None, - [Leaf(token.NUMBER, 1), Leaf(token.NUMBER, 2), - Leaf(token.NUMBER, 3)], - [Leaf(token.NUMBER, 1), Leaf(token.NUMBER, 3), - Leaf(token.NUMBER, 2), Leaf(token.NUMBER, 4)], - [Leaf(token.STRING, "b"), Leaf(token.STRING, "j", prefix=" ")] - ] - self.assertStr(self._Call("A"), "A()") - self.assertStr(self._Call("b", kids[1]), "b(1,2,3)") - self.assertStr(self._Call("a.b().c", kids[2]), "a.b().c(1,3,2,4)") - self.assertStr(self._Call("d", kids[3], prefix=" "), " d(b, j)") - - -class Test_does_tree_import(support.TestCase): - def _find_bind_rec(self, name, node): - # Search a tree for a binding -- used to find the starting - # point for these tests. - c = fixer_util.find_binding(name, node) - if c: return c - for child in node.children: - c = self._find_bind_rec(name, child) - if c: return c - - def does_tree_import(self, package, name, string): - node = parse(string) - # Find the binding of start -- that's what we'll go from - node = self._find_bind_rec('start', node) - return fixer_util.does_tree_import(package, name, node) - - def try_with(self, string): - failing_tests = (("a", "a", "from a import b"), - ("a.d", "a", "from a.d import b"), - ("d.a", "a", "from d.a import b"), - (None, "a", "import b"), - (None, "a", "import b, c, d")) - for package, name, import_ in failing_tests: - n = self.does_tree_import(package, name, import_ + "\n" + string) - self.assertFalse(n) - n = self.does_tree_import(package, name, string + "\n" + import_) - self.assertFalse(n) - - passing_tests = (("a", "a", "from a import a"), - ("x", "a", "from x import a"), - ("x", "a", "from x import b, c, a, d"), - ("x.b", "a", "from x.b import a"), - ("x.b", "a", "from x.b import b, c, a, d"), - (None, "a", "import a"), - (None, "a", "import b, c, a, d")) - for package, name, import_ in passing_tests: - n = self.does_tree_import(package, name, import_ + "\n" + string) - self.assertTrue(n) - n = self.does_tree_import(package, name, string + "\n" + import_) - self.assertTrue(n) - - def test_in_function(self): - self.try_with("def foo():\n\tbar.baz()\n\tstart=3") - -class Test_find_binding(support.TestCase): - def find_binding(self, name, string, package=None): - return fixer_util.find_binding(name, parse(string), package) - - def test_simple_assignment(self): - self.assertTrue(self.find_binding("a", "a = b")) - self.assertTrue(self.find_binding("a", "a = [b, c, d]")) - self.assertTrue(self.find_binding("a", "a = foo()")) - self.assertTrue(self.find_binding("a", "a = foo().foo.foo[6][foo]")) - self.assertFalse(self.find_binding("a", "foo = a")) - self.assertFalse(self.find_binding("a", "foo = (a, b, c)")) - - def test_tuple_assignment(self): - self.assertTrue(self.find_binding("a", "(a,) = b")) - self.assertTrue(self.find_binding("a", "(a, b, c) = [b, c, d]")) - self.assertTrue(self.find_binding("a", "(c, (d, a), b) = foo()")) - self.assertTrue(self.find_binding("a", "(a, b) = foo().foo[6][foo]")) - self.assertFalse(self.find_binding("a", "(foo, b) = (b, a)")) - self.assertFalse(self.find_binding("a", "(foo, (b, c)) = (a, b, c)")) - - def test_list_assignment(self): - self.assertTrue(self.find_binding("a", "[a] = b")) - self.assertTrue(self.find_binding("a", "[a, b, c] = [b, c, d]")) - self.assertTrue(self.find_binding("a", "[c, [d, a], b] = foo()")) - self.assertTrue(self.find_binding("a", "[a, b] = foo().foo[a][foo]")) - self.assertFalse(self.find_binding("a", "[foo, b] = (b, a)")) - self.assertFalse(self.find_binding("a", "[foo, [b, c]] = (a, b, c)")) - - def test_invalid_assignments(self): - self.assertFalse(self.find_binding("a", "foo.a = 5")) - self.assertFalse(self.find_binding("a", "foo[a] = 5")) - self.assertFalse(self.find_binding("a", "foo(a) = 5")) - self.assertFalse(self.find_binding("a", "foo(a, b) = 5")) - - def test_simple_import(self): - self.assertTrue(self.find_binding("a", "import a")) - self.assertTrue(self.find_binding("a", "import b, c, a, d")) - self.assertFalse(self.find_binding("a", "import b")) - self.assertFalse(self.find_binding("a", "import b, c, d")) - - def test_from_import(self): - self.assertTrue(self.find_binding("a", "from x import a")) - self.assertTrue(self.find_binding("a", "from a import a")) - self.assertTrue(self.find_binding("a", "from x import b, c, a, d")) - self.assertTrue(self.find_binding("a", "from x.b import a")) - self.assertTrue(self.find_binding("a", "from x.b import b, c, a, d")) - self.assertFalse(self.find_binding("a", "from a import b")) - self.assertFalse(self.find_binding("a", "from a.d import b")) - self.assertFalse(self.find_binding("a", "from d.a import b")) - - def test_import_as(self): - self.assertTrue(self.find_binding("a", "import b as a")) - self.assertTrue(self.find_binding("a", "import b as a, c, a as f, d")) - self.assertFalse(self.find_binding("a", "import a as f")) - self.assertFalse(self.find_binding("a", "import b, c as f, d as e")) - - def test_from_import_as(self): - self.assertTrue(self.find_binding("a", "from x import b as a")) - self.assertTrue(self.find_binding("a", "from x import g as a, d as b")) - self.assertTrue(self.find_binding("a", "from x.b import t as a")) - self.assertTrue(self.find_binding("a", "from x.b import g as a, d")) - self.assertFalse(self.find_binding("a", "from a import b as t")) - self.assertFalse(self.find_binding("a", "from a.d import b as t")) - self.assertFalse(self.find_binding("a", "from d.a import b as t")) - - def test_simple_import_with_package(self): - self.assertTrue(self.find_binding("b", "import b")) - self.assertTrue(self.find_binding("b", "import b, c, d")) - self.assertFalse(self.find_binding("b", "import b", "b")) - self.assertFalse(self.find_binding("b", "import b, c, d", "c")) - - def test_from_import_with_package(self): - self.assertTrue(self.find_binding("a", "from x import a", "x")) - self.assertTrue(self.find_binding("a", "from a import a", "a")) - self.assertTrue(self.find_binding("a", "from x import *", "x")) - self.assertTrue(self.find_binding("a", "from x import b, c, a, d", "x")) - self.assertTrue(self.find_binding("a", "from x.b import a", "x.b")) - self.assertTrue(self.find_binding("a", "from x.b import *", "x.b")) - self.assertTrue(self.find_binding("a", "from x.b import b, c, a, d", "x.b")) - self.assertFalse(self.find_binding("a", "from a import b", "a")) - self.assertFalse(self.find_binding("a", "from a.d import b", "a.d")) - self.assertFalse(self.find_binding("a", "from d.a import b", "a.d")) - self.assertFalse(self.find_binding("a", "from x.y import *", "a.b")) - - def test_import_as_with_package(self): - self.assertFalse(self.find_binding("a", "import b.c as a", "b.c")) - self.assertFalse(self.find_binding("a", "import a as f", "f")) - self.assertFalse(self.find_binding("a", "import a as f", "a")) - - def test_from_import_as_with_package(self): - # Because it would take a lot of special-case code in the fixers - # to deal with from foo import bar as baz, we'll simply always - # fail if there is an "from ... import ... as ..." - self.assertFalse(self.find_binding("a", "from x import b as a", "x")) - self.assertFalse(self.find_binding("a", "from x import g as a, d as b", "x")) - self.assertFalse(self.find_binding("a", "from x.b import t as a", "x.b")) - self.assertFalse(self.find_binding("a", "from x.b import g as a, d", "x.b")) - self.assertFalse(self.find_binding("a", "from a import b as t", "a")) - self.assertFalse(self.find_binding("a", "from a import b as t", "b")) - self.assertFalse(self.find_binding("a", "from a import b as t", "t")) - - def test_function_def(self): - self.assertTrue(self.find_binding("a", "def a(): pass")) - self.assertTrue(self.find_binding("a", "def a(b, c, d): pass")) - self.assertTrue(self.find_binding("a", "def a(): b = 7")) - self.assertFalse(self.find_binding("a", "def d(b, (c, a), e): pass")) - self.assertFalse(self.find_binding("a", "def d(a=7): pass")) - self.assertFalse(self.find_binding("a", "def d(a): pass")) - self.assertFalse(self.find_binding("a", "def d(): a = 7")) - - s = """ - def d(): - def a(): - pass""" - self.assertFalse(self.find_binding("a", s)) - - def test_class_def(self): - self.assertTrue(self.find_binding("a", "class a: pass")) - self.assertTrue(self.find_binding("a", "class a(): pass")) - self.assertTrue(self.find_binding("a", "class a(b): pass")) - self.assertTrue(self.find_binding("a", "class a(b, c=8): pass")) - self.assertFalse(self.find_binding("a", "class d: pass")) - self.assertFalse(self.find_binding("a", "class d(a): pass")) - self.assertFalse(self.find_binding("a", "class d(b, a=7): pass")) - self.assertFalse(self.find_binding("a", "class d(b, *a): pass")) - self.assertFalse(self.find_binding("a", "class d(b, **a): pass")) - self.assertFalse(self.find_binding("a", "class d: a = 7")) - - s = """ - class d(): - class a(): - pass""" - self.assertFalse(self.find_binding("a", s)) - - def test_for(self): - self.assertTrue(self.find_binding("a", "for a in r: pass")) - self.assertTrue(self.find_binding("a", "for a, b in r: pass")) - self.assertTrue(self.find_binding("a", "for (a, b) in r: pass")) - self.assertTrue(self.find_binding("a", "for c, (a,) in r: pass")) - self.assertTrue(self.find_binding("a", "for c, (a, b) in r: pass")) - self.assertTrue(self.find_binding("a", "for c in r: a = c")) - self.assertFalse(self.find_binding("a", "for c in a: pass")) - - def test_for_nested(self): - s = """ - for b in r: - for a in b: - pass""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - for b in r: - for a, c in b: - pass""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - for b in r: - for (a, c) in b: - pass""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - for b in r: - for (a,) in b: - pass""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - for b in r: - for c, (a, d) in b: - pass""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - for b in r: - for c in b: - a = 7""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - for b in r: - for c in b: - d = a""" - self.assertFalse(self.find_binding("a", s)) - - s = """ - for b in r: - for c in a: - d = 7""" - self.assertFalse(self.find_binding("a", s)) - - def test_if(self): - self.assertTrue(self.find_binding("a", "if b in r: a = c")) - self.assertFalse(self.find_binding("a", "if a in r: d = e")) - - def test_if_nested(self): - s = """ - if b in r: - if c in d: - a = c""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - if b in r: - if c in d: - c = a""" - self.assertFalse(self.find_binding("a", s)) - - def test_while(self): - self.assertTrue(self.find_binding("a", "while b in r: a = c")) - self.assertFalse(self.find_binding("a", "while a in r: d = e")) - - def test_while_nested(self): - s = """ - while b in r: - while c in d: - a = c""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - while b in r: - while c in d: - c = a""" - self.assertFalse(self.find_binding("a", s)) - - def test_try_except(self): - s = """ - try: - a = 6 - except: - b = 8""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - try: - b = 8 - except: - a = 6""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - try: - b = 8 - except KeyError: - pass - except: - a = 6""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - try: - b = 8 - except: - b = 6""" - self.assertFalse(self.find_binding("a", s)) - - def test_try_except_nested(self): - s = """ - try: - try: - a = 6 - except: - pass - except: - b = 8""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - try: - b = 8 - except: - try: - a = 6 - except: - pass""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - try: - b = 8 - except: - try: - pass - except: - a = 6""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - try: - try: - b = 8 - except KeyError: - pass - except: - a = 6 - except: - pass""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - try: - pass - except: - try: - b = 8 - except KeyError: - pass - except: - a = 6""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - try: - b = 8 - except: - b = 6""" - self.assertFalse(self.find_binding("a", s)) - - s = """ - try: - try: - b = 8 - except: - c = d - except: - try: - b = 6 - except: - t = 8 - except: - o = y""" - self.assertFalse(self.find_binding("a", s)) - - def test_try_except_finally(self): - s = """ - try: - c = 6 - except: - b = 8 - finally: - a = 9""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - try: - b = 8 - finally: - a = 6""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - try: - b = 8 - finally: - b = 6""" - self.assertFalse(self.find_binding("a", s)) - - s = """ - try: - b = 8 - except: - b = 9 - finally: - b = 6""" - self.assertFalse(self.find_binding("a", s)) - - def test_try_except_finally_nested(self): - s = """ - try: - c = 6 - except: - b = 8 - finally: - try: - a = 9 - except: - b = 9 - finally: - c = 9""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - try: - b = 8 - finally: - try: - pass - finally: - a = 6""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - try: - b = 8 - finally: - try: - b = 6 - finally: - b = 7""" - self.assertFalse(self.find_binding("a", s)) - -class Test_touch_import(support.TestCase): - - def test_after_docstring(self): - node = parse('"""foo"""\nbar()') - fixer_util.touch_import(None, "foo", node) - self.assertEqual(str(node), '"""foo"""\nimport foo\nbar()\n\n') - - def test_after_imports(self): - node = parse('"""foo"""\nimport bar\nbar()') - fixer_util.touch_import(None, "foo", node) - self.assertEqual(str(node), '"""foo"""\nimport bar\nimport foo\nbar()\n\n') - - def test_beginning(self): - node = parse('bar()') - fixer_util.touch_import(None, "foo", node) - self.assertEqual(str(node), 'import foo\nbar()\n\n') - - def test_from_import(self): - node = parse('bar()') - fixer_util.touch_import("html", "escape", node) - self.assertEqual(str(node), 'from html import escape\nbar()\n\n') - - def test_name_import(self): - node = parse('bar()') - fixer_util.touch_import(None, "cgi", node) - self.assertEqual(str(node), 'import cgi\nbar()\n\n') - -class Test_find_indentation(support.TestCase): - - def test_nothing(self): - fi = fixer_util.find_indentation - node = parse("node()") - self.assertEqual(fi(node), "") - node = parse("") - self.assertEqual(fi(node), "") - - def test_simple(self): - fi = fixer_util.find_indentation - node = parse("def f():\n x()") - self.assertEqual(fi(node), "") - self.assertEqual(fi(node.children[0].children[4].children[2]), " ") - node = parse("def f():\n x()\n y()") - self.assertEqual(fi(node.children[0].children[4].children[4]), " ") diff --git a/Lib/test/test_tools/test_sundry.py b/Lib/test/test_tools/test_sundry.py index 3177fafb84a65b..2f8ba272164d32 100644 --- a/Lib/test/test_tools/test_sundry.py +++ b/Lib/test/test_tools/test_sundry.py @@ -14,11 +14,6 @@ skip_if_missing() class TestSundryScripts(unittest.TestCase): - # At least make sure the rest don't have syntax errors. When tests are - # added for a script it should be added to the allowlist below. - - skiplist = ['2to3'] - # import logging registers "atfork" functions which keep indirectly the # logging module dictionary alive. Mock the function to be able to unload # cleanly the logging module. @@ -31,9 +26,6 @@ def test_sundry(self, mock_os): continue name = fn[:-3] - if name in self.skiplist: - continue - import_tool(name) finally: # Unload all modules loaded in this test diff --git a/Mac/BuildScript/scripts/postflight.framework b/Mac/BuildScript/scripts/postflight.framework index a63909358f1719..411fda13b89559 100755 --- a/Mac/BuildScript/scripts/postflight.framework +++ b/Mac/BuildScript/scripts/postflight.framework @@ -8,12 +8,12 @@ FWK="/Library/Frameworks/Python.framework/Versions/@PYVER@" "${FWK}/bin/python@PYVER@" -E -s -Wi \ "${FWK}/lib/python${PYVER}/compileall.py" -q -j0 \ - -f -x 'bad_coding|badsyntax|site-packages|test/test_lib2to3/data' \ + -f -x 'bad_coding|badsyntax|site-packages' \ "${FWK}/lib/python${PYVER}" "${FWK}/bin/python@PYVER@" -E -s -Wi -O \ "${FWK}/lib/python${PYVER}/compileall.py" -q -j0 \ - -f -x 'bad_coding|badsyntax|site-packages|test/test_lib2to3/data' \ + -f -x 'bad_coding|badsyntax|site-packages' \ "${FWK}/lib/python${PYVER}" "${FWK}/bin/python@PYVER@" -E -s -Wi \ diff --git a/Mac/Makefile.in b/Mac/Makefile.in index 69ab4198988570..78b4499cca986a 100644 --- a/Mac/Makefile.in +++ b/Mac/Makefile.in @@ -63,7 +63,6 @@ installunixtools: fi cd "$(DESTDIR)$(FRAMEWORKUNIXTOOLSPREFIX)/bin" && \ for fn in \ - 2to3 \ idle3 \ pydoc3 \ python3 \ @@ -123,7 +122,6 @@ altinstallunixtools: fi cd "$(DESTDIR)$(FRAMEWORKUNIXTOOLSPREFIX)/bin" && \ for fn in \ - 2to3-$(VERSION) \ idle$(VERSION) \ pydoc$(VERSION) \ python$(VERSION) \ diff --git a/Makefile.pre.in b/Makefile.pre.in index eb79c9c4ca1801..091df3e4c0b814 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1869,7 +1869,7 @@ hostrunnertest: all pythoninfo: all $(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test.pythoninfo -QUICKTESTOPTS= $(TESTOPTS) -x test_subprocess test_io test_lib2to3 \ +QUICKTESTOPTS= $(TESTOPTS) -x test_subprocess test_io \ test_multibytecodec test_urllib2_localnet test_itertools \ test_multiprocessing_fork test_multiprocessing_spawn \ test_multiprocessing_forkserver \ @@ -2042,8 +2042,6 @@ bininstall: altbininstall (cd $(DESTDIR)$(BINDIR); $(LN) -s idle$(VERSION) idle3) -rm -f $(DESTDIR)$(BINDIR)/pydoc3 (cd $(DESTDIR)$(BINDIR); $(LN) -s pydoc$(VERSION) pydoc3) - -rm -f $(DESTDIR)$(BINDIR)/2to3 - (cd $(DESTDIR)$(BINDIR); $(LN) -s 2to3-$(VERSION) 2to3) if test "x$(LIPO_32BIT_FLAGS)" != "x" ; then \ rm -f $(DESTDIR)$(BINDIR)/python3-32$(EXE); \ (cd $(DESTDIR)$(BINDIR); $(LN) -s python$(VERSION)-32$(EXE) python3-32$(EXE)) \ @@ -2090,7 +2088,6 @@ LIBSUBDIRS= asyncio \ idlelib idlelib/Icons \ importlib importlib/resources importlib/metadata \ json \ - lib2to3 lib2to3/fixes lib2to3/pgen2 \ logging \ multiprocessing multiprocessing/dummy \ pydoc_data \ @@ -2184,10 +2181,6 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_importlib/resources/zipdata02 \ test/test_importlib/source \ test/test_json \ - test/test_lib2to3 \ - test/test_lib2to3/data \ - test/test_lib2to3/data/fixers \ - test/test_lib2to3/data/fixers/myfixes \ test/test_peg_generator \ test/test_sqlite3 \ test/test_tkinter \ @@ -2301,16 +2294,12 @@ libinstall: all $(srcdir)/Modules/xxmodule.c -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ $(PYTHON_FOR_BUILD) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \ -o 0 -o 1 -o 2 $(COMPILEALL_OPTS) -d $(LIBDEST) -f \ - -x 'bad_coding|badsyntax|site-packages|test/test_lib2to3/data' \ + -x 'bad_coding|badsyntax|site-packages' \ $(DESTDIR)$(LIBDEST) -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ $(PYTHON_FOR_BUILD) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \ -o 0 -o 1 -o 2 $(COMPILEALL_OPTS) -d $(LIBDEST)/site-packages -f \ -x badsyntax $(DESTDIR)$(LIBDEST)/site-packages - -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ - $(PYTHON_FOR_BUILD) -m lib2to3.pgen2.driver $(DESTDIR)$(LIBDEST)/lib2to3/Grammar.txt - -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ - $(PYTHON_FOR_BUILD) -m lib2to3.pgen2.driver $(DESTDIR)$(LIBDEST)/lib2to3/PatternGrammar.txt # bpo-21536: Misc/python-config.sh is generated in the build directory # from $(srcdir)Misc/python-config.sh.in. @@ -2330,15 +2319,9 @@ python-config: $(srcdir)/Misc/python-config.in Misc/python-config.sh # macOS' make seems to ignore a dependency on a # "$(BUILD_SCRIPTS_DIR): $(MKDIR_P) $@" rule. BUILD_SCRIPTS_DIR=build/scripts-$(VERSION) -SCRIPT_2TO3=$(BUILD_SCRIPTS_DIR)/2to3-$(VERSION) SCRIPT_IDLE=$(BUILD_SCRIPTS_DIR)/idle$(VERSION) SCRIPT_PYDOC=$(BUILD_SCRIPTS_DIR)/pydoc$(VERSION) -$(SCRIPT_2TO3): $(srcdir)/Tools/scripts/2to3 - @$(MKDIR_P) $(BUILD_SCRIPTS_DIR) - sed -e "s,/usr/bin/env python3,$(EXENAME)," < $(srcdir)/Tools/scripts/2to3 > $@ - @chmod +x $@ - $(SCRIPT_IDLE): $(srcdir)/Tools/scripts/idle3 @$(MKDIR_P) $(BUILD_SCRIPTS_DIR) sed -e "s,/usr/bin/env python3,$(EXENAME)," < $(srcdir)/Tools/scripts/idle3 > $@ @@ -2350,7 +2333,7 @@ $(SCRIPT_PYDOC): $(srcdir)/Tools/scripts/pydoc3 @chmod +x $@ .PHONY: scripts -scripts: $(SCRIPT_2TO3) $(SCRIPT_IDLE) $(SCRIPT_PYDOC) python-config +scripts: $(SCRIPT_IDLE) $(SCRIPT_PYDOC) python-config # Install the include files INCLDIRSTOMAKE=$(INCLUDEDIR) $(CONFINCLUDEDIR) $(INCLUDEPY) $(CONFINCLUDEPY) @@ -2436,7 +2419,6 @@ libainstall: all scripts $(INSTALL_SCRIPT) $(srcdir)/install-sh $(DESTDIR)$(LIBPL)/install-sh $(INSTALL_SCRIPT) python-config.py $(DESTDIR)$(LIBPL)/python-config.py $(INSTALL_SCRIPT) python-config $(DESTDIR)$(BINDIR)/python$(LDVERSION)-config - $(INSTALL_SCRIPT) $(SCRIPT_2TO3) $(DESTDIR)$(BINDIR)/2to3-$(VERSION) $(INSTALL_SCRIPT) $(SCRIPT_IDLE) $(DESTDIR)$(BINDIR)/idle$(VERSION) $(INSTALL_SCRIPT) $(SCRIPT_PYDOC) $(DESTDIR)$(BINDIR)/pydoc$(VERSION) @if [ -s Modules/python.exp -a \ @@ -2637,7 +2619,6 @@ clean-retain-profile: pycremoval find build -name '*.py' -exec rm -f {} ';' || true find build -name '*.py[co]' -exec rm -f {} ';' || true -rm -f pybuilddir.txt - -rm -f Lib/lib2to3/*Grammar*.pickle -rm -f _bootstrap_python -rm -f python.html python*.js python.data python*.symbols python*.map -rm -f $(WASM_STDLIB) diff --git a/Misc/NEWS.d/3.11.0a1.rst b/Misc/NEWS.d/3.11.0a1.rst index 7670e482ede5b6..10e123e325c863 100644 --- a/Misc/NEWS.d/3.11.0a1.rst +++ b/Misc/NEWS.d/3.11.0a1.rst @@ -1825,7 +1825,7 @@ Patch by Victor Stinner. .. nonce: 9nmMtB .. section: Library -The :mod:`lib2to3` package is now deprecated and may not be able to parse +The :mod:`!lib2to3` package is now deprecated and may not be able to parse Python 3.10 or newer. See the :pep:`617` (New PEG parser for CPython). Patch by Victor Stinner. diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst index 854458f2d1a994..2564167cdf0895 100644 --- a/Misc/NEWS.d/3.8.0a1.rst +++ b/Misc/NEWS.d/3.8.0a1.rst @@ -2747,7 +2747,7 @@ smaller file). Also, change the default compression level to 6 (tradeoff). .. nonce: xo5LAr .. section: Library -The :term:`2to3` :2to3fixer:`execfile` fixer now opens the file with mode +The 2to3 ``execfile`` fixer now opens the file with mode ``'rb'``. Patch by Zackery Spytz. .. @@ -3817,7 +3817,7 @@ user. .. nonce: w6g7tn .. section: Library -The :2to3fixer:`reload` fixer now uses :func:`importlib.reload` instead of +The ``reload`` fixer now uses :func:`importlib.reload` instead of deprecated :func:`!imp.reload`. .. diff --git a/Misc/NEWS.d/3.9.0a3.rst b/Misc/NEWS.d/3.9.0a3.rst index 54b61ca3b7785f..b05b4c3d7204c0 100644 --- a/Misc/NEWS.d/3.9.0a3.rst +++ b/Misc/NEWS.d/3.9.0a3.rst @@ -571,7 +571,7 @@ new task spawning before exception raising. .. section: Library Correctly parenthesize filter-based statements that contain lambda -expressions in mod:`lib2to3`. Patch by Dong-hee Na. +expressions in mod:`!lib2to3`. Patch by Dong-hee Na. .. diff --git a/Misc/NEWS.d/3.9.0a6.rst b/Misc/NEWS.d/3.9.0a6.rst index af2cc7c3e97889..9594964917f390 100644 --- a/Misc/NEWS.d/3.9.0a6.rst +++ b/Misc/NEWS.d/3.9.0a6.rst @@ -343,7 +343,7 @@ https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=242274 .. nonce: Er8sv- .. section: Library -The :mod:`lib2to3` module is pending deprecation due to :pep:`617`. +The :mod:`!lib2to3` module is pending deprecation due to :pep:`617`. .. diff --git a/Misc/NEWS.d/next/Library/2023-05-23-03-36-47.gh-issue-104780.P4e3Yf.rst b/Misc/NEWS.d/next/Library/2023-05-23-03-36-47.gh-issue-104780.P4e3Yf.rst new file mode 100644 index 00000000000000..acdca53b4bb7d4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-23-03-36-47.gh-issue-104780.P4e3Yf.rst @@ -0,0 +1,2 @@ +Remove the ``2to3`` program and the :mod:`!lib2to3` module, deprecated in +Python 3.11. Patch by Victor Stinner. diff --git a/PC/layout/main.py b/PC/layout/main.py index c9246007d47d18..cb2e4878da26b1 100644 --- a/PC/layout/main.py +++ b/PC/layout/main.py @@ -49,8 +49,6 @@ REQUIRED_DLLS = FileStemSet("libcrypto*", "libssl*", "libffi*") -LIB2TO3_GRAMMAR_FILES = FileNameSet("Grammar.txt", "PatternGrammar.txt") - PY_FILES = FileSuffixSet(".py") PYC_FILES = FileSuffixSet(".pyc") CAT_FILES = FileSuffixSet(".cat") @@ -298,27 +296,6 @@ def _write_to_zip(zf, dest, src, ns, checked=True): log_exception("Failed to delete {}", pyc) return - if src in LIB2TO3_GRAMMAR_FILES: - from lib2to3.pgen2.driver import load_grammar - - tmp = ns.temp / src.name - try: - shutil.copy(src, tmp) - load_grammar(str(tmp)) - for f in ns.temp.glob(src.stem + "*.pickle"): - zf.write(str(f), str(dest.parent / f.name)) - try: - f.unlink() - except: - log_exception("Failed to delete {}", f) - except: - log_exception("Failed to compile {}", src) - finally: - try: - tmp.unlink() - except: - log_exception("Failed to delete {}", tmp) - zf.write(str(src), str(dest)) diff --git a/PC/layout/support/props.py b/PC/layout/support/props.py index c7a7a0ce777a6e..f751a4b3f1e437 100644 --- a/PC/layout/support/props.py +++ b/PC/layout/support/props.py @@ -36,7 +36,6 @@ {PYTHON_VERSION} true - false false {PYTHON_TARGET};$(GetPythonRuntimeFilesDependsOn) @@ -67,7 +66,6 @@ DLLs\%(Filename)%(Extension) <_PythonRuntimeLib Include="$(PythonHome)\Lib\**\*" Exclude="$(PythonHome)\Lib\**\*.pyc;$(PythonHome)\Lib\site-packages\**\*" /> - <_PythonRuntimeLib Remove="$(PythonHome)\Lib\lib2to3\**\*" Condition="$(IncludeLib2To3) != 'true'" /> <_PythonRuntimeLib Remove="$(PythonHome)\Lib\ensurepip\**\*" Condition="$(IncludeVEnv) != 'true'" /> <_PythonRuntimeLib Remove="$(PythonHome)\Lib\venv\**\*" Condition="$(IncludeVEnv) != 'true'" /> <_PythonRuntimeLib> diff --git a/PC/layout/support/python.props b/PC/layout/support/python.props index e46891aafcb9fc..c48ecb2f9c1b57 100644 --- a/PC/layout/support/python.props +++ b/PC/layout/support/python.props @@ -8,7 +8,6 @@ $$PYTHON_VERSION$$ true - false false $$PYTHON_TARGET$$;$(GetPythonRuntimeFilesDependsOn) @@ -40,7 +39,6 @@ DLLs\%(Filename)%(Extension) <_PythonRuntimeLib Include="$(PythonHome)\Lib\**\*" Exclude="$(PythonHome)\Lib\**\*.pyc;$(PythonHome)\Lib\site-packages\**\*" /> - <_PythonRuntimeLib Remove="$(PythonHome)\Lib\lib2to3\**\*" Condition="$(IncludeLib2To3) != 'true'" /> <_PythonRuntimeLib Remove="$(PythonHome)\Lib\ensurepip\**\*" Condition="$(IncludeVEnv) != 'true'" /> <_PythonRuntimeLib Remove="$(PythonHome)\Lib\venv\**\*" Condition="$(IncludeVEnv) != 'true'" /> <_PythonRuntimeLib> diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 4b0580557fb7a4..b926caf4a57907 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -172,7 +172,6 @@ static const char* _Py_stdlib_module_names[] = { "itertools", "json", "keyword", -"lib2to3", "linecache", "locale", "logging", diff --git a/Tools/README b/Tools/README index e51624f453c5bb..9c4b6d86e990ba 100644 --- a/Tools/README +++ b/Tools/README @@ -39,10 +39,8 @@ patchcheck Tools for checking and applying patches to the Python source cod peg_generator PEG-based parser generator (pegen) used for new parser. -scripts A number of useful single-file programs, e.g. tabnanny.py - by Tim Peters, which checks for inconsistent mixing of - tabs and spaces, and 2to3, which converts Python 2 code - to Python 3 code. +scripts A number of useful single-file programs, e.g. run_tests.py + which runs the Python test suite. ssl Scripts to generate ssl_data.h from OpenSSL sources, and run tests against multiple installations of OpenSSL and LibreSSL. diff --git a/Tools/msi/bundle/packagegroups/postinstall.wxs b/Tools/msi/bundle/packagegroups/postinstall.wxs index 64f42dd30e8ba0..24c72e6357ab08 100644 --- a/Tools/msi/bundle/packagegroups/postinstall.wxs +++ b/Tools/msi/bundle/packagegroups/postinstall.wxs @@ -42,7 +42,7 @@ - + - - - - diff --git a/Tools/msi/msi.props b/Tools/msi/msi.props index 06aa0b8bbca277..cfb3ca9e76e24c 100644 --- a/Tools/msi/msi.props +++ b/Tools/msi/msi.props @@ -156,9 +156,6 @@ <_Uuid Include="PythonwExeComponentGuid"> pythonw.exe - <_Uuid Include="RemoveLib2to3PickleComponentGuid"> - lib2to3/pickles - <_Uuid Include="CommonPythonRegComponentGuid"> registry @@ -187,4 +184,4 @@ $(DefineConstants);@(_UuidValue,';'); - \ No newline at end of file + diff --git a/Tools/peg_generator/Makefile b/Tools/peg_generator/Makefile index 084da154919e3e..eef2d807e5c5ce 100644 --- a/Tools/peg_generator/Makefile +++ b/Tools/peg_generator/Makefile @@ -87,8 +87,7 @@ time_stdlib: $(CPYTHON) venv $(VENVPYTHON) scripts/test_parse_directory.py \ -d $(CPYTHON) \ $(TESTFLAGS) \ - --exclude "*/bad*" \ - --exclude "*/test/test_lib2to3/data/*" + --exclude "*/bad*" mypy: regen-metaparser $(MYPY) # For list of files, see mypy.ini diff --git a/Tools/peg_generator/scripts/benchmark.py b/Tools/peg_generator/scripts/benchmark.py index 053f8ef06d42c5..6354e1a17167af 100644 --- a/Tools/peg_generator/scripts/benchmark.py +++ b/Tools/peg_generator/scripts/benchmark.py @@ -78,7 +78,6 @@ def run_benchmark_stdlib(subcommand): verbose=False, excluded_files=[ "*/bad*", - "*/test/test_lib2to3/data/*", ], short=True, mode=modes[subcommand], diff --git a/Tools/scripts/2to3 b/Tools/scripts/2to3 deleted file mode 100755 index f27d18ecf67086..00000000000000 --- a/Tools/scripts/2to3 +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 -import sys -from lib2to3.main import main - -sys.exit(main("lib2to3.fixes")) diff --git a/Tools/scripts/README b/Tools/scripts/README index 9dbb89a8dae633..a078bfbf662a37 100644 --- a/Tools/scripts/README +++ b/Tools/scripts/README @@ -1,7 +1,6 @@ This directory contains a collection of executable Python scripts that are useful while building, extending or managing Python. -2to3 Main script for running the 2to3 conversion tool checkpip.py Checks the version of the projects bundled in ensurepip are the latest available combinerefs.py A helper for analyzing PYTHONDUMPREFS output diff --git a/Tools/wasm/wasm_assets.py b/Tools/wasm/wasm_assets.py index 22bf9eabdcd6b4..fcd99405ee0d8e 100755 --- a/Tools/wasm/wasm_assets.py +++ b/Tools/wasm/wasm_assets.py @@ -41,8 +41,6 @@ # package management "ensurepip/", "venv/", - # build system - "lib2to3/", # deprecated "uu.py", "xdrlib.py", From 13b5d79090cdca138ca340de88f57e02279faf6d Mon Sep 17 00:00:00 2001 From: chgnrdv <52372310+chgnrdv@users.noreply.github.com> Date: Tue, 23 May 2023 23:01:17 +0300 Subject: [PATCH 40/47] Fix missing/incomplete NULL checks in multiple source files (#104564) Co-authored-by: Oleg Iarygin --- Modules/_zoneinfo.c | 10 +++++++++- Modules/errnomodule.c | 1 + Modules/posixmodule.c | 4 ++++ Modules/sha1module.c | 3 +++ Modules/zlibmodule.c | 3 +++ 5 files changed, 20 insertions(+), 1 deletion(-) diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index c8c791b6d7c0d8..0dcdb4da47d5db 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -2381,7 +2381,12 @@ get_local_timestamp(PyObject *dt, int64_t *local_ts) ///// // Functions for cache handling -/* Constructor for StrongCacheNode */ +/* Constructor for StrongCacheNode + * + * This function doesn't set MemoryError if PyMem_Malloc fails, + * as the cache intentionally doesn't propagate exceptions + * and fails silently if error occurs. + */ static StrongCacheNode * strong_cache_node_new(PyObject *key, PyObject *zone) { @@ -2572,6 +2577,9 @@ update_strong_cache(zoneinfo_state *state, const PyTypeObject *const type, } StrongCacheNode *new_node = strong_cache_node_new(key, zone); + if (new_node == NULL) { + return; + } StrongCacheNode **root = &(state->ZONEINFO_STRONG_CACHE); move_strong_cache_node_to_front(state, root, new_node); diff --git a/Modules/errnomodule.c b/Modules/errnomodule.c index fddde960a5fe9a..3d0c2d7ae945bc 100644 --- a/Modules/errnomodule.c +++ b/Modules/errnomodule.c @@ -84,6 +84,7 @@ errno_exec(PyObject *module) PyObject *module_dict = PyModule_GetDict(module); PyObject *error_dict = PyDict_New(); if (!module_dict || !error_dict) { + Py_XDECREF(error_dict); return -1; } if (PyDict_SetItemString(module_dict, "errorcode", error_dict) < 0) { diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 531f26ba8bc86f..2fe9973ef5bf7a 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -9023,6 +9023,10 @@ os_setgroups(PyObject *module, PyObject *groups) } gid_t *grouplist = PyMem_New(gid_t, len); + if (grouplist == NULL) { + PyErr_NoMemory(); + return NULL; + } for (Py_ssize_t i = 0; i < len; i++) { PyObject *elem; elem = PySequence_GetItem(groups, i); diff --git a/Modules/sha1module.c b/Modules/sha1module.c index c66269b5f5cdf3..ef8e067dd337b3 100644 --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -73,6 +73,9 @@ static SHA1object * newSHA1object(SHA1State *st) { SHA1object *sha = (SHA1object *)PyObject_GC_New(SHA1object, st->sha1_type); + if (sha == NULL) { + return NULL; + } sha->lock = NULL; PyObject_GC_Track(sha); return sha; diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index b67844a67c315c..534d065765f0f9 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -1722,6 +1722,9 @@ ZlibDecompressor__new__(PyTypeObject *cls, return NULL; } ZlibDecompressor *self = PyObject_New(ZlibDecompressor, cls); + if (self == NULL) { + return NULL; + } self->eof = 0; self->needs_input = 1; self->avail_in_real = 0; From 08b4eb83aadcbdb389b5970b51cac9be95146c2a Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Tue, 23 May 2023 16:03:13 -0400 Subject: [PATCH 41/47] Remove gh-103207 changelog item as it was never part of any release. (GH-104814) --- Misc/NEWS.d/3.12.0b1.rst | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/Misc/NEWS.d/3.12.0b1.rst b/Misc/NEWS.d/3.12.0b1.rst index eb42b45726f5ed..a1ea082b3a2119 100644 --- a/Misc/NEWS.d/3.12.0b1.rst +++ b/Misc/NEWS.d/3.12.0b1.rst @@ -2236,16 +2236,6 @@ update curses textbox to additionally handle backspace using the .. -.. date: 2023-04-04-13-37-28 -.. gh-issue: 103207 -.. nonce: x0vvQp -.. section: macOS - -Add instructions to the macOS installer welcome display on how to workaround -the macOS 13 Ventura “The installer encountered an error” failure. - -.. - .. date: 2023-03-24-11-20-47 .. gh-issue: 102997 .. nonce: ZgQkbq From 097b7830cd67f039ff36ba4fa285d82d26e25e84 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 23 May 2023 14:29:30 -0600 Subject: [PATCH 42/47] gh-104341: Add a Separate "Running" Lock for Each Thread (gh-104754) Having a separate lock means Thread.join() doesn't need to wait for the thread to be cleaned up first. It can wait for the thread's Python target to finish running. This gives us some flexibility in how we clean up threads. (This is a minor cleanup as part of a fix for gh-104341.) --- Lib/test/test_threading.py | 24 ++++++------- Lib/threading.py | 70 ++++++++++++++++++++++---------------- 2 files changed, 53 insertions(+), 41 deletions(-) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 97165264b34bbe..648533923dcd81 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -747,7 +747,7 @@ def f(): rc, out, err = assert_python_ok("-c", code) self.assertEqual(err, b"") - def test_tstate_lock(self): + def test_running_lock(self): # Test an implementation detail of Thread objects. started = _thread.allocate_lock() finish = _thread.allocate_lock() @@ -757,29 +757,29 @@ def f(): started.release() finish.acquire() time.sleep(0.01) - # The tstate lock is None until the thread is started + # The running lock is None until the thread is started t = threading.Thread(target=f) - self.assertIs(t._tstate_lock, None) + self.assertIs(t._running_lock, None) t.start() started.acquire() self.assertTrue(t.is_alive()) - # The tstate lock can't be acquired when the thread is running + # The running lock can't be acquired when the thread is running # (or suspended). - tstate_lock = t._tstate_lock - self.assertFalse(tstate_lock.acquire(timeout=0), False) + running_lock = t._running_lock + self.assertFalse(running_lock.acquire(timeout=0), False) finish.release() # When the thread ends, the state_lock can be successfully # acquired. - self.assertTrue(tstate_lock.acquire(timeout=support.SHORT_TIMEOUT), False) - # But is_alive() is still True: we hold _tstate_lock now, which - # prevents is_alive() from knowing the thread's end-of-life C code + self.assertTrue(running_lock.acquire(timeout=support.SHORT_TIMEOUT), False) + # But is_alive() is still True: we hold _running_lock now, which + # prevents is_alive() from knowing the thread's Python code # is done. self.assertTrue(t.is_alive()) # Let is_alive() find out the C code is done. - tstate_lock.release() + running_lock.release() self.assertFalse(t.is_alive()) - # And verify the thread disposed of _tstate_lock. - self.assertIsNone(t._tstate_lock) + # And verify the thread disposed of _running_lock. + self.assertIsNone(t._running_lock) t.join() def test_repr_stopped(self): diff --git a/Lib/threading.py b/Lib/threading.py index df273870fa4273..69b4f54c050adf 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -908,6 +908,7 @@ class is implemented. self._ident = None if _HAVE_THREAD_NATIVE_ID: self._native_id = None + self._running_lock = None self._tstate_lock = None self._started = Event() self._is_stopped = False @@ -926,6 +927,9 @@ def _reset_internal_locks(self, is_alive): # bpo-42350: If the fork happens when the thread is already stopped # (ex: after threading._shutdown() has been called), _tstate_lock # is None. Do nothing in this case. + if self._running_lock is not None: + self._running_lock._at_fork_reinit() + self._running_lock.acquire() if self._tstate_lock is not None: self._tstate_lock._at_fork_reinit() self._tstate_lock.acquire() @@ -933,6 +937,7 @@ def _reset_internal_locks(self, is_alive): # The thread isn't alive after fork: it doesn't have a tstate # anymore. self._is_stopped = True + self._running_lock = None self._tstate_lock = None def __repr__(self): @@ -1019,6 +1024,14 @@ def _set_ident(self): def _set_native_id(self): self._native_id = get_native_id() + def _set_running_lock(self): + """ + Set a lock object which will be released by the interpreter when + the target func has finished running. + """ + self._running_lock = _allocate_lock() + self._running_lock.acquire() + def _set_tstate_lock(self): """ Set a lock object which will be released by the interpreter when @@ -1035,6 +1048,7 @@ def _set_tstate_lock(self): def _bootstrap_inner(self): try: self._set_ident() + self._set_running_lock() self._set_tstate_lock() if _HAVE_THREAD_NATIVE_ID: self._set_native_id() @@ -1054,29 +1068,29 @@ def _bootstrap_inner(self): self._invoke_excepthook(self) finally: self._delete() + self._running_lock.release() def _stop(self): # After calling ._stop(), .is_alive() returns False and .join() returns - # immediately. ._tstate_lock must be released before calling ._stop(). + # immediately. ._running_lock must be released before calling ._stop(). # - # Normal case: C code at the end of the thread's life - # (release_sentinel in _threadmodule.c) releases ._tstate_lock, and - # that's detected by our ._wait_for_tstate_lock(), called by .join() + # Normal case: ._bootstrap_inner() releases ._running_lock, and + # that's detected by our ._wait_for_running_lock(), called by .join() # and .is_alive(). Any number of threads _may_ call ._stop() # simultaneously (for example, if multiple threads are blocked in # .join() calls), and they're not serialized. That's harmless - # they'll just make redundant rebindings of ._is_stopped and - # ._tstate_lock. Obscure: we rebind ._tstate_lock last so that the - # "assert self._is_stopped" in ._wait_for_tstate_lock() always works - # (the assert is executed only if ._tstate_lock is None). + # ._running_lock. Obscure: we rebind ._running_lock last so that the + # "assert self._is_stopped" in ._wait_for_running_lock() always works + # (the assert is executed only if ._running_lock is None). # - # Special case: _main_thread releases ._tstate_lock via this + # Special case: _main_thread releases ._running_lock via this # module's _shutdown() function. - lock = self._tstate_lock + lock = self._running_lock if lock is not None: assert not lock.locked() self._is_stopped = True - self._tstate_lock = None + self._running_lock = None if not self.daemon: with _shutdown_locks_lock: # Remove our lock and other released locks from _shutdown_locks @@ -1123,20 +1137,17 @@ def join(self, timeout=None): raise RuntimeError("cannot join current thread") if timeout is None: - self._wait_for_tstate_lock() + self._wait_for_running_lock() else: # the behavior of a negative timeout isn't documented, but # historically .join(timeout=x) for x<0 has acted as if timeout=0 - self._wait_for_tstate_lock(timeout=max(timeout, 0)) - - def _wait_for_tstate_lock(self, block=True, timeout=-1): - # Issue #18808: wait for the thread state to be gone. - # At the end of the thread's life, after all knowledge of the thread - # is removed from C data structures, C code releases our _tstate_lock. - # This method passes its arguments to _tstate_lock.acquire(). - # If the lock is acquired, the C code is done, and self._stop() is - # called. That sets ._is_stopped to True, and ._tstate_lock to None. - lock = self._tstate_lock + self._wait_for_running_lock(timeout=max(timeout, 0)) + + def _wait_for_running_lock(self, block=True, timeout=-1): + # This method passes its arguments to _running_lock.acquire(). + # If the lock is acquired, the python code is done, and self._stop() is + # called. That sets ._is_stopped to True, and ._running_lock to None. + lock = self._running_lock if lock is None: # already determined that the C code is done assert self._is_stopped @@ -1207,7 +1218,7 @@ def is_alive(self): assert self._initialized, "Thread.__init__() not called" if self._is_stopped or not self._started.is_set(): return False - self._wait_for_tstate_lock(False) + self._wait_for_running_lock(False) return not self._is_stopped @property @@ -1417,7 +1428,7 @@ class _MainThread(Thread): def __init__(self): Thread.__init__(self, name="MainThread", daemon=False) - self._set_tstate_lock() + self._set_running_lock() self._started.set() self._set_ident() if _HAVE_THREAD_NATIVE_ID: @@ -1558,7 +1569,7 @@ def _shutdown(): # dubious, but some code does it. We can't wait for C code to release # the main thread's tstate_lock - that won't happen until the interpreter # is nearly dead. So we release it here. Note that just calling _stop() - # isn't enough: other threads may already be waiting on _tstate_lock. + # isn't enough: other threads may already be waiting on _running_lock. if _main_thread._is_stopped: # _shutdown() was already called return @@ -1573,12 +1584,13 @@ def _shutdown(): # Main thread if _main_thread.ident == get_ident(): - tlock = _main_thread._tstate_lock - # The main thread isn't finished yet, so its thread state lock can't + assert _main_thread._tstate_lock is None + running_lock = _main_thread._running_lock + # The main thread isn't finished yet, so its running lock can't # have been released. - assert tlock is not None - assert tlock.locked() - tlock.release() + assert running_lock is not None + assert running_lock.locked() + running_lock.release() _main_thread._stop() else: # bpo-1596321: _shutdown() must be called in the main thread. From 50fce89d123b25e53fa8a0303a169e8887154a0e Mon Sep 17 00:00:00 2001 From: Robert O'Shea Date: Tue, 23 May 2023 21:44:40 +0100 Subject: [PATCH 43/47] gh-102120: [TarFile] Add an iter function that doesn't cache (GH-102128) --- Doc/library/tarfile.rst | 7 +++++- Lib/tarfile.py | 17 +++++++++----- Lib/test/test_tarfile.py | 23 +++++++++++++++++++ ...-03-08-19-30-53.gh-issue-102120.xkQ5Wr.rst | 2 ++ 4 files changed, 42 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-08-19-30-53.gh-issue-102120.xkQ5Wr.rst diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 891af1bcf7edff..2f330f018a48be 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -318,7 +318,7 @@ be finalized; only the internally used file object will be closed. See the .. versionadded:: 3.2 Added support for the context management protocol. -.. class:: TarFile(name=None, mode='r', fileobj=None, format=DEFAULT_FORMAT, tarinfo=TarInfo, dereference=False, ignore_zeros=False, encoding=ENCODING, errors='surrogateescape', pax_headers=None, debug=0, errorlevel=1) +.. class:: TarFile(name=None, mode='r', fileobj=None, format=DEFAULT_FORMAT, tarinfo=TarInfo, dereference=False, ignore_zeros=False, encoding=ENCODING, errors='surrogateescape', pax_headers=None, debug=0, errorlevel=1, stream=False) All following arguments are optional and can be accessed as instance attributes as well. @@ -369,6 +369,9 @@ be finalized; only the internally used file object will be closed. See the The *pax_headers* argument is an optional dictionary of strings which will be added as a pax global header if *format* is :const:`PAX_FORMAT`. + If *stream* is set to :const:`True` then while reading the archive info about files + in the archive are not cached, saving memory. + .. versionchanged:: 3.2 Use ``'surrogateescape'`` as the default for the *errors* argument. @@ -378,6 +381,8 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.6 The *name* parameter accepts a :term:`path-like object`. + .. versionchanged:: 3.13 + Add the *stream* parameter. .. classmethod:: TarFile.open(...) diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 7781a430839ea5..df4e41f7a0d23a 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -1633,7 +1633,7 @@ class TarFile(object): def __init__(self, name=None, mode="r", fileobj=None, format=None, tarinfo=None, dereference=None, ignore_zeros=None, encoding=None, errors="surrogateescape", pax_headers=None, debug=None, - errorlevel=None, copybufsize=None): + errorlevel=None, copybufsize=None, stream=False): """Open an (uncompressed) tar archive `name'. `mode' is either 'r' to read from an existing archive, 'a' to append data to an existing file or 'w' to create a new file overwriting an existing one. `mode' @@ -1665,6 +1665,8 @@ def __init__(self, name=None, mode="r", fileobj=None, format=None, self.name = os.path.abspath(name) if name else None self.fileobj = fileobj + self.stream = stream + # Init attributes. if format is not None: self.format = format @@ -2631,7 +2633,9 @@ def next(self): break if tarinfo is not None: - self.members.append(tarinfo) + # if streaming the file we do not want to cache the tarinfo + if not self.stream: + self.members.append(tarinfo) else: self._loaded = True @@ -2682,11 +2686,12 @@ def _getmember(self, name, tarinfo=None, normalize=False): def _load(self): """Read through the entire archive file and look for readable - members. + members. This should not run if the file is set to stream. """ - while self.next() is not None: - pass - self._loaded = True + if not self.stream: + while self.next() is not None: + pass + self._loaded = True def _check(self, mode=None): """Check if TarFile is still open, and if the operation's mode diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index e8d322d20a5a8e..2eda7fc4ceac71 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -100,6 +100,14 @@ def setUp(self): def tearDown(self): self.tar.close() +class StreamModeTest(ReadTest): + + # Only needs to change how the tarfile is opened to set + # stream mode + def setUp(self): + self.tar = tarfile.open(self.tarname, mode=self.mode, + encoding="iso8859-1", + stream=True) class UstarReadTest(ReadTest, unittest.TestCase): @@ -852,6 +860,21 @@ class Bz2StreamReadTest(Bz2Test, StreamReadTest): class LzmaStreamReadTest(LzmaTest, StreamReadTest): pass +class TarStreamModeReadTest(StreamModeTest, unittest.TestCase): + + def test_stream_mode_no_cache(self): + for _ in self.tar: + pass + self.assertEqual(self.tar.members, []) + +class GzipStreamModeReadTest(GzipTest, TarStreamModeReadTest): + pass + +class Bz2StreamModeReadTest(Bz2Test, TarStreamModeReadTest): + pass + +class LzmaStreamModeReadTest(LzmaTest, TarStreamModeReadTest): + pass class DetectReadTest(TarTest, unittest.TestCase): def _testfunc_file(self, name, mode): diff --git a/Misc/NEWS.d/next/Library/2023-03-08-19-30-53.gh-issue-102120.xkQ5Wr.rst b/Misc/NEWS.d/next/Library/2023-03-08-19-30-53.gh-issue-102120.xkQ5Wr.rst new file mode 100644 index 00000000000000..ca50242fdbe293 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-08-19-30-53.gh-issue-102120.xkQ5Wr.rst @@ -0,0 +1,2 @@ +Added a stream mode to ``tarfile`` that allows for reading +archives without caching info about the inner files. From d56c933992c86986bd58eb3880aed0ed1b0cadc9 Mon Sep 17 00:00:00 2001 From: Nicolas Tessore Date: Tue, 23 May 2023 21:51:56 +0100 Subject: [PATCH 44/47] gh-104770: Let generator.close() return value (#104771) Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> --- Doc/reference/expressions.rst | 19 +++-- Lib/test/test_generators.py | 82 +++++++++++++++++++ ...-05-23-00-36-02.gh-issue-104770.poSkyY.rst | 2 + Objects/genobject.c | 11 ++- 4 files changed, 105 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-23-00-36-02.gh-issue-104770.poSkyY.rst diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index b97a08f25d92a2..0c700f908d6878 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -595,12 +595,19 @@ is already executing raises a :exc:`ValueError` exception. .. method:: generator.close() Raises a :exc:`GeneratorExit` at the point where the generator function was - paused. If the generator function then exits gracefully, is already closed, - or raises :exc:`GeneratorExit` (by not catching the exception), close - returns to its caller. If the generator yields a value, a - :exc:`RuntimeError` is raised. If the generator raises any other exception, - it is propagated to the caller. :meth:`close` does nothing if the generator - has already exited due to an exception or normal exit. + paused. If the generator function catches the exception and returns a + value, this value is returned from :meth:`close`. If the generator function + is already closed, or raises :exc:`GeneratorExit` (by not catching the + exception), :meth:`close` returns :const:`None`. If the generator yields a + value, a :exc:`RuntimeError` is raised. If the generator raises any other + exception, it is propagated to the caller. If the generator has already + exited due to an exception or normal exit, :meth:`close` returns + :const:`None` and has no other effect. + + .. versionchanged:: 3.13 + + If a generator returns a value upon being closed, the value is returned + by :meth:`close`. .. index:: single: yield; examples diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index 31680b5a92e0f3..a8a344ab8de48d 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -451,6 +451,88 @@ def g(): self.assertEqual(cm.exception.value.value, 2) +class GeneratorCloseTest(unittest.TestCase): + + def test_close_no_return_value(self): + def f(): + yield + + gen = f() + gen.send(None) + self.assertIsNone(gen.close()) + + def test_close_return_value(self): + def f(): + try: + yield + # close() raises GeneratorExit here, which is caught + except GeneratorExit: + return 0 + + gen = f() + gen.send(None) + self.assertEqual(gen.close(), 0) + + def test_close_not_catching_exit(self): + def f(): + yield + # close() raises GeneratorExit here, which isn't caught and + # therefore propagates -- no return value + return 0 + + gen = f() + gen.send(None) + self.assertIsNone(gen.close()) + + def test_close_not_started(self): + def f(): + try: + yield + except GeneratorExit: + return 0 + + gen = f() + self.assertIsNone(gen.close()) + + def test_close_exhausted(self): + def f(): + try: + yield + except GeneratorExit: + return 0 + + gen = f() + next(gen) + with self.assertRaises(StopIteration): + next(gen) + self.assertIsNone(gen.close()) + + def test_close_closed(self): + def f(): + try: + yield + except GeneratorExit: + return 0 + + gen = f() + gen.send(None) + self.assertEqual(gen.close(), 0) + self.assertIsNone(gen.close()) + + def test_close_raises(self): + def f(): + try: + yield + except GeneratorExit: + pass + raise RuntimeError + + gen = f() + gen.send(None) + with self.assertRaises(RuntimeError): + gen.close() + + class GeneratorThrowTest(unittest.TestCase): def test_exception_context_with_yield(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-23-00-36-02.gh-issue-104770.poSkyY.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-23-00-36-02.gh-issue-104770.poSkyY.rst new file mode 100644 index 00000000000000..2103fb7d61c21a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-23-00-36-02.gh-issue-104770.poSkyY.rst @@ -0,0 +1,2 @@ +If a generator returns a value upon being closed, the value is now returned +by :meth:`generator.close`. diff --git a/Objects/genobject.c b/Objects/genobject.c index 9252c654934565..1abfc83ab678ef 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -408,11 +408,16 @@ gen_close(PyGenObject *gen, PyObject *args) PyErr_SetString(PyExc_RuntimeError, msg); return NULL; } - if (PyErr_ExceptionMatches(PyExc_StopIteration) - || PyErr_ExceptionMatches(PyExc_GeneratorExit)) { - PyErr_Clear(); /* ignore these errors */ + assert(PyErr_Occurred()); + if (PyErr_ExceptionMatches(PyExc_GeneratorExit)) { + PyErr_Clear(); /* ignore this error */ Py_RETURN_NONE; } + /* if the generator returned a value while closing, StopIteration was + * raised in gen_send_ex() above; retrieve and return the value here */ + if (_PyGen_FetchStopIterationValue(&retval) == 0) { + return retval; + } return NULL; } From e0b3078705b271ff278dfbc788c2b061c92a9aa3 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Tue, 23 May 2023 16:04:31 -0600 Subject: [PATCH 45/47] gh-103295: fix stack overwrite on 32-bit in perf map test harness (#104811) --- Modules/_testinternalcapi.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index b91f7b620fdb4e..8267dbf6779017 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -762,19 +762,24 @@ clear_extension(PyObject *self, PyObject *args) static PyObject * write_perf_map_entry(PyObject *self, PyObject *args) { + PyObject *code_addr_v; const void *code_addr; unsigned int code_size; const char *entry_name; - if (!PyArg_ParseTuple(args, "KIs", &code_addr, &code_size, &entry_name)) + if (!PyArg_ParseTuple(args, "OIs", &code_addr_v, &code_size, &entry_name)) return NULL; + code_addr = PyLong_AsVoidPtr(code_addr_v); + if (code_addr == NULL) { + return NULL; + } int ret = PyUnstable_WritePerfMapEntry(code_addr, code_size, entry_name); - if (ret == -1) { - PyErr_SetString(PyExc_OSError, "Failed to write performance map entry"); + if (ret < 0) { + PyErr_SetFromErrno(PyExc_OSError); return NULL; } - return Py_BuildValue("i", ret); + return PyLong_FromLong(ret); } static PyObject * From 6b1510cf11c16c8e4381810c15ceeda6f89e79f4 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Tue, 23 May 2023 23:31:59 +0100 Subject: [PATCH 46/47] GH-83863: Drop support for using `pathlib.Path` objects as context managers (GH-104807) In Python 3.8 and prior, `pathlib.Path.__exit__()` marked a path as closed; some subsequent attempts to perform I/O would raise an IOError. This functionality was never documented, and had the effect of making `Path` objects mutable, contrary to PEP 428. In Python 3.9 we made `__exit__()` a no-op, and in 3.11 `__enter__()` began raising deprecation warnings. Here we remove both methods. --- Doc/whatsnew/3.13.rst | 3 +++ Lib/pathlib.py | 19 ------------------ Lib/test/test_pathlib.py | 20 ------------------- ...3-05-23-19-53-18.gh-issue-83863.eRI5JG.rst | 4 ++++ 4 files changed, 7 insertions(+), 39 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-05-23-19-53-18.gh-issue-83863.eRI5JG.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index e0c3c2a3592ec7..309f26ea27dbfd 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -115,6 +115,9 @@ Removed are now removed. The items in those namespaces can be imported directly from :mod:`typing`. (Contributed by Sebastian Rittau in :gh:`92871`.) +* Remove support for using :class:`pathlib.Path` objects as context managers. + This functionality was deprecated and made a no-op in Python 3.9. + Porting to Python 3.13 ====================== diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 3d68c161603d08..3a7a1241ba77f6 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -1080,25 +1080,6 @@ def __new__(cls, *args, **kwargs): cls = WindowsPath if os.name == 'nt' else PosixPath return object.__new__(cls) - def __enter__(self): - # In previous versions of pathlib, __exit__() marked this path as - # closed; subsequent attempts to perform I/O would raise an IOError. - # This functionality was never documented, and had the effect of - # making Path objects mutable, contrary to PEP 428. - # In Python 3.9 __exit__() was made a no-op. - # In Python 3.11 __enter__() began emitting DeprecationWarning. - # In Python 3.13 __enter__() and __exit__() should be removed. - warnings.warn("pathlib.Path.__enter__() is deprecated and scheduled " - "for removal in Python 3.13; Path objects as a context " - "manager is a no-op", - DeprecationWarning, stacklevel=2) - return self - - def __exit__(self, t, v, tb): - pass - - # Public API - @classmethod def cwd(cls): """Return a new path pointing to the current working directory.""" diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index ab2c2b232a0411..8b68cdc9b7d003 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -2080,26 +2080,6 @@ def test_resolve_nonexist_relative_issue38671(self): finally: os.chdir(old_cwd) - def test_with(self): - p = self.cls(BASE) - it = p.iterdir() - it2 = p.iterdir() - next(it2) - # bpo-46556: path context managers are deprecated in Python 3.11. - with self.assertWarns(DeprecationWarning): - with p: - pass - # Using a path as a context manager is a no-op, thus the following - # operations should still succeed after the context manage exits. - next(it) - next(it2) - p.exists() - p.resolve() - p.absolute() - with self.assertWarns(DeprecationWarning): - with p: - pass - @os_helper.skip_unless_working_chmod def test_chmod(self): p = self.cls(BASE) / 'fileA' diff --git a/Misc/NEWS.d/next/Library/2023-05-23-19-53-18.gh-issue-83863.eRI5JG.rst b/Misc/NEWS.d/next/Library/2023-05-23-19-53-18.gh-issue-83863.eRI5JG.rst new file mode 100644 index 00000000000000..7a073aa37dd6ae --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-23-19-53-18.gh-issue-83863.eRI5JG.rst @@ -0,0 +1,4 @@ +Support for using :class:`pathlib.Path` objects as context managers has been +removed. Before Python 3.9, exiting the context manager marked a path as +"closed", which caused some (but not all!) methods to raise when called. +Since Python 3.9, using a path as a context manager does nothing. From 2e0931046dcc200fd6abb2cdfaf57d8b99117c57 Mon Sep 17 00:00:00 2001 From: melanie witt Date: Tue, 23 May 2023 16:10:34 -0700 Subject: [PATCH 47/47] gh-85934: Use getattr_static when adding mock spec (#22209) Co-authored-by: Terry Jan Reedy Co-authored-by: Oleg Iarygin --- Lib/test/test_unittest/testmock/testmock.py | 31 +++++++++++++++++++ Lib/unittest/mock.py | 8 ++++- .../2020-09-16-16-53-06.bpo-41768.8_fWkC.rst | 2 ++ 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2020-09-16-16-53-06.bpo-41768.8_fWkC.rst diff --git a/Lib/test/test_unittest/testmock/testmock.py b/Lib/test/test_unittest/testmock/testmock.py index d1cae47a40eed4..c7895e73ad9a8c 100644 --- a/Lib/test/test_unittest/testmock/testmock.py +++ b/Lib/test/test_unittest/testmock/testmock.py @@ -38,6 +38,17 @@ def cmeth(cls, a, b, c, d=None): pass def smeth(a, b, c, d=None): pass +class SomethingElse(object): + def __init__(self): + self._instance = None + + @property + def instance(self): + if not self._instance: + self._instance = 'object' + return self._instance + + class Typos(): autospect = None auto_spec = None @@ -2293,6 +2304,26 @@ class Foo(): f'{__name__}.Typos', autospect=True, set_spec=True, auto_spec=True): pass + def test_property_not_called_with_spec_mock(self): + obj = SomethingElse() + self.assertIsNone(obj._instance, msg='before mock') + mock = Mock(spec=obj) + self.assertIsNone(obj._instance, msg='after mock') + self.assertEqual('object', obj.instance) + + def test_decorated_async_methods_with_spec_mock(self): + class Foo(): + @classmethod + async def class_method(cls): + pass + @staticmethod + async def static_method(): + pass + async def method(self): + pass + mock = Mock(spec=Foo) + for m in (mock.method, mock.class_method, mock.static_method): + self.assertIsInstance(m, AsyncMock) if __name__ == '__main__': unittest.main() diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 7ca085760650af..22f81e55b567f2 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -526,7 +526,13 @@ def _mock_add_spec(self, spec, spec_set, _spec_as_instance=False, spec_list = dir(spec) for attr in spec_list: - if iscoroutinefunction(getattr(spec, attr, None)): + static_attr = inspect.getattr_static(spec, attr, None) + unwrapped_attr = static_attr + try: + unwrapped_attr = inspect.unwrap(unwrapped_attr) + except ValueError: + pass + if iscoroutinefunction(unwrapped_attr): _spec_asyncs.append(attr) spec = spec_list diff --git a/Misc/NEWS.d/next/Library/2020-09-16-16-53-06.bpo-41768.8_fWkC.rst b/Misc/NEWS.d/next/Library/2020-09-16-16-53-06.bpo-41768.8_fWkC.rst new file mode 100644 index 00000000000000..bfd3a294d44efa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-09-16-16-53-06.bpo-41768.8_fWkC.rst @@ -0,0 +1,2 @@ +:mod:`unittest.mock` speccing no longer calls class properties. +Patch by Melanie Witt.