From f6161df9be85bda6cc604939486b6dcccdbc850b Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 24 Sep 2024 17:29:43 -0700 Subject: [PATCH 1/4] gh-119180: Make FORWARDREF format look at __annotations__ first From discussion with @larryhastings and @carljm, this is the desired behavior. --- Lib/annotationlib.py | 81 ++++++++++++++++++++++--------- Lib/test/test_annotationlib.py | 88 ++++++++++++++++++++++++++++++++-- 2 files changed, 143 insertions(+), 26 deletions(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 0a67742a2b3081..abc82213e044e6 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -639,28 +639,38 @@ def get_annotations( if eval_str and format != Format.VALUE: raise ValueError("eval_str=True is only supported with format=Format.VALUE") - # For VALUE format, we look at __annotations__ directly. - if format != Format.VALUE: - annotate = get_annotate_function(obj) - if annotate is not None: - ann = call_annotate_function(annotate, format, owner=obj) - if not isinstance(ann, dict): - raise ValueError(f"{obj!r}.__annotate__ returned a non-dict") - return dict(ann) - - if isinstance(obj, type): - try: - ann = _BASE_GET_ANNOTATIONS(obj) - except AttributeError: - # For static types, the descriptor raises AttributeError. - return {} - else: - ann = getattr(obj, "__annotations__", None) - if ann is None: - return {} - - if not isinstance(ann, dict): - raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") + match format: + case Format.VALUE: + # For VALUE, we only look at __annotations__ + ann = _get_dunder_annotations(obj) + case Format.FORWARDREF: + # For FORWARDREF, we use __annotations__ if it exists + try: + ann = _get_dunder_annotations(obj) + except NameError: + pass + else: + return dict(ann) + + # But if __annotations__ threw a NameError, we try calling __annotate__ + ann = _get_and_call_annotate(obj, format) + if ann is not None: + return ann + + # If that didn't work either, we have a very weird object: evaluating + # __annotations__ threw NameError and there is no __annotate__. In that case, + # we fall back to trying __annotations__ again. + return dict(_get_dunder_annotations(obj)) + case Format.SOURCE: + # For SOURCE, we try to call __annotate__ + ann = _get_and_call_annotate(obj, format) + if ann is not None: + return ann + # But if we didn't get it, we use __annotations__ instead. + ann = _get_dunder_annotations(obj) + return ann + case _: + raise ValueError(f"Unsupported format {format!r}") if not ann: return {} @@ -725,3 +735,30 @@ def get_annotations( for key, value in ann.items() } return return_value + + +def _get_and_call_annotate(obj, format): + annotate = get_annotate_function(obj) + if annotate is not None: + ann = call_annotate_function(annotate, format, owner=obj) + if not isinstance(ann, dict): + raise ValueError(f"{obj!r}.__annotate__ returned a non-dict") + return dict(ann) + return None + + +def _get_dunder_annotations(obj): + if isinstance(obj, type): + try: + ann = _BASE_GET_ANNOTATIONS(obj) + except AttributeError: + # For static types, the descriptor raises AttributeError. + return {} + else: + ann = getattr(obj, "__annotations__", None) + if ann is None: + return {} + + if not isinstance(ann, dict): + raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") + return dict(ann) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index dd8ceb55a411fb..62f57c0a83d4e2 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -705,17 +705,97 @@ def f(x: int): self.assertEqual(annotationlib.get_annotations(f), {"x": int}) self.assertEqual( - annotationlib.get_annotations(f, format=annotationlib.Format.FORWARDREF), + annotationlib.get_annotations(f, format=Format.FORWARDREF), {"x": int}, ) f.__annotations__["x"] = str # The modification is reflected in VALUE (the default) self.assertEqual(annotationlib.get_annotations(f), {"x": str}) - # ... but not in FORWARDREF, which uses __annotate__ + # ... and also in FORWARDREF, which tries __annotations__ if available self.assertEqual( - annotationlib.get_annotations(f, format=annotationlib.Format.FORWARDREF), - {"x": int}, + annotationlib.get_annotations(f, format=Format.FORWARDREF), + {"x": str}, + ) + # ... but not in SOURCE which always uses __annotate__ + self.assertEqual( + annotationlib.get_annotations(f, format=Format.SOURCE), + {"x": "int"}, + ) + + def test_non_dict_annotations(self): + class WeirdAnnotations: + @property + def __annotations__(self): + return "not a dict" + + wa = WeirdAnnotations() + for format in Format: + with ( + self.subTest(format=format), + self.assertRaisesRegex( + ValueError, r".*__annotations__ is neither a dict nor None" + ), + ): + annotationlib.get_annotations(wa, format=format) + + def test_annotations_on_custom_object(self): + class HasAnnotations: + @property + def __annotations__(self): + return {"x": int} + + ha = HasAnnotations() + self.assertEqual( + annotationlib.get_annotations(ha, format=Format.VALUE), {"x": int} + ) + self.assertEqual( + annotationlib.get_annotations(ha, format=Format.FORWARDREF), {"x": int} + ) + + # TODO(gh-124412): This should return {'x': 'int'} instead. + self.assertEqual( + annotationlib.get_annotations(ha, format=Format.SOURCE), {"x": int} + ) + + def test_raising_annotations_on_custom_object(self): + class HasRaisingAnnotations: + @property + def __annotations__(self): + return {"x": undefined} + + hra = HasRaisingAnnotations() + + with self.assertRaises(NameError): + annotationlib.get_annotations(hra, format=Format.VALUE) + + with self.assertRaises(NameError): + annotationlib.get_annotations(hra, format=Format.FORWARDREF) + + undefined = float + self.assertEqual( + annotationlib.get_annotations(hra, format=Format.VALUE), {"x": float} + ) + + def test_forwardref_prefers_annotations(self): + class HasBoth: + @property + def __annotations__(self): + return {"x": int} + + @property + def __annotate__(self): + return lambda format: {"x": str} + + hb = HasBoth() + self.assertEqual( + annotationlib.get_annotations(hb, format=Format.VALUE), {"x": int} + ) + self.assertEqual( + annotationlib.get_annotations(hb, format=Format.FORWARDREF), {"x": int} + ) + self.assertEqual( + annotationlib.get_annotations(hb, format=Format.SOURCE), {"x": str} ) def test_pep695_generic_class_with_future_annotations(self): From 430f184f8abf55da8ae3b877e9a1d13302148ab5 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 25 Sep 2024 14:50:26 -0700 Subject: [PATCH 2/4] gh-124412: Add helpers for converting annotations to source format --- Doc/library/annotationlib.rst | 34 ++++++++++++++++++++++++++++++++++ Lib/_collections_abc.py | 22 +++------------------- Lib/annotationlib.py | 28 +++++++++++++++++++++++++++- Lib/test/test_annotationlib.py | 33 ++++++++++++++++++++++++++++++--- Lib/typing.py | 21 +++------------------ 5 files changed, 97 insertions(+), 41 deletions(-) diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index 1e72c5421674bc..3e03db101e1096 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -197,6 +197,27 @@ Classes Functions --------- +.. function:: annotations_to_source(annotations) + + Convert an annotations dict containing runtime values to a + dict containing only strings. If the values are not already strings, + they are converted using :func:`value_to_source`. + This is meant as a helper for user-provided + annotate functions that support the :attr:`~Format.SOURCE` format but + do not have access to the code creating the annotations. + + For example, this is used to implement the :attr:`~Format.SOURCE` for + :class:`typing.TypedDict` classes created through the functional syntax: + + .. doctest:: + + >>> from typing import TypedDict + >>> Movie = TypedDict("movie", {"name": str, "year": int}) + >>> get_annotations(Movie, format=Format.SOURCE) + {'name': 'str', 'year': 'int'} + + .. versionadded:: 3.14 + .. function:: call_annotate_function(annotate, format, *, owner=None) Call the :term:`annotate function` *annotate* with the given *format*, @@ -347,3 +368,16 @@ Functions {'a': , 'b': , 'return': } .. versionadded:: 3.14 + +.. function:: value_to_source(value) + + Convert an arbitrary Python value to a format suitable for use by the + :attr:`~Format.SOURCE` format. This calls :func:`repr` for most + objects, but has special handling for some objects, such as type objects. + + This is meant as a helper for user-provided + annotate functions that support the :attr:`~Format.SOURCE` format but + do not have access to the code creating the annotations. + + .. versionadded:: 3.14 + diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 75252b3a87f9c4..4139cbadf93e13 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -485,9 +485,10 @@ def __new__(cls, origin, args): def __repr__(self): if len(self.__args__) == 2 and _is_param_expr(self.__args__[0]): return super().__repr__() + from annotationlib import value_to_source return (f'collections.abc.Callable' - f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], ' - f'{_type_repr(self.__args__[-1])}]') + f'[[{", ".join([value_to_source(a) for a in self.__args__[:-1]])}], ' + f'{value_to_source(self.__args__[-1])}]') def __reduce__(self): args = self.__args__ @@ -524,23 +525,6 @@ def _is_param_expr(obj): names = ('ParamSpec', '_ConcatenateGenericAlias') return obj.__module__ == 'typing' and any(obj.__name__ == name for name in names) -def _type_repr(obj): - """Return the repr() of an object, special-casing types (internal helper). - - Copied from :mod:`typing` since collections.abc - shouldn't depend on that module. - (Keep this roughly in sync with the typing version.) - """ - if isinstance(obj, type): - if obj.__module__ == 'builtins': - return obj.__qualname__ - return f'{obj.__module__}.{obj.__qualname__}' - if obj is Ellipsis: - return '...' - if isinstance(obj, FunctionType): - return obj.__name__ - return repr(obj) - class Callable(metaclass=ABCMeta): diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 20c9542efac2d8..b465b1f3ab9245 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -15,6 +15,8 @@ "call_evaluate_function", "get_annotate_function", "get_annotations", + "annotations_to_source", + "value_to_source", ] @@ -693,7 +695,7 @@ def get_annotations( return ann # But if we didn't get it, we use __annotations__ instead. ann = _get_dunder_annotations(obj) - return ann + return annotations_to_source(ann) case _: raise ValueError(f"Unsupported format {format!r}") @@ -762,6 +764,30 @@ def get_annotations( return return_value +def value_to_source(value): + """Convert a Python value to a format suitable for use with the SOURCE format. + + This is inteded as a helper for tools that support the SOURCE format but do + not have access to the code that originally produced the annotations. It uses + repr() for most objects. + + """ + if isinstance(value, type): + if value.__module__ == 'builtins': + return value.__qualname__ + return f'{value.__module__}.{value.__qualname__}' + if value is ...: + return '...' + if isinstance(value, (types.FunctionType, types.BuiltinFunctionType)): + return value.__name__ + return repr(value) + + +def annotations_to_source(annotations): + """Convert an annotation dict containing values to approximately the SOURCE format.""" + return {n: t if isinstance(t, str) else value_to_source(t) for n, t in annotations.items()} + + def _get_and_call_annotate(obj, format): annotate = get_annotate_function(obj) if annotate is not None: diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 5b052dab5007d6..180627e774b6da 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -7,7 +7,7 @@ import itertools import pickle import unittest -from annotationlib import Format, ForwardRef, get_annotations, get_annotate_function +from annotationlib import Format, ForwardRef, get_annotations, get_annotate_function, annotations_to_source, value_to_source from typing import Unpack from test import support @@ -25,6 +25,11 @@ def wrapper(a, b): return wrapper +class MyClass: + def __repr__(self): + return "my repr" + + class TestFormat(unittest.TestCase): def test_enum(self): self.assertEqual(annotationlib.Format.VALUE.value, 1) @@ -788,9 +793,8 @@ def __annotations__(self): annotationlib.get_annotations(ha, format=Format.FORWARDREF), {"x": int} ) - # TODO(gh-124412): This should return {'x': 'int'} instead. self.assertEqual( - annotationlib.get_annotations(ha, format=Format.SOURCE), {"x": int} + annotationlib.get_annotations(ha, format=Format.SOURCE), {"x": "int"} ) def test_raising_annotations_on_custom_object(self): @@ -1078,6 +1082,29 @@ class C: self.assertEqual(get_annotate_function(C)(Format.VALUE), {"a": int}) +class TestToSource(unittest.TestCase): + def test_value_to_source(self): + self.assertEqual(value_to_source(int), "int") + self.assertEqual(value_to_source(MyClass), "test.test_annotationlib.MyClass") + self.assertEqual(value_to_source(len), "len") + self.assertEqual(value_to_source(value_to_source), "value_to_source") + self.assertEqual(value_to_source(times_three), "times_three") + self.assertEqual(value_to_source(...), "...") + self.assertEqual(value_to_source(None), "None") + self.assertEqual(value_to_source(1), "1") + self.assertEqual(value_to_source("1"), "'1'") + self.assertEqual(value_to_source(Format.VALUE), repr(Format.VALUE)) + self.assertEqual(value_to_source(MyClass()), "my repr") + + def test_annotations_to_source(self): + self.assertEqual(annotations_to_source({}), {}) + self.assertEqual(annotations_to_source({"x": int}), {"x": "int"}) + self.assertEqual(annotations_to_source({"x": "int"}), {"x": "int"}) + self.assertEqual(annotations_to_source({"x": int, "y": str}), {"x": "int", "y": "str"}) + + class TestAnnotationLib(unittest.TestCase): def test__all__(self): support.check__all__(self, annotationlib) + + diff --git a/Lib/typing.py b/Lib/typing.py index 9377e771d60f4b..252eef32cd88a4 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -242,21 +242,10 @@ def _type_repr(obj): typically enough to uniquely identify a type. For everything else, we fall back on repr(obj). """ - # When changing this function, don't forget about - # `_collections_abc._type_repr`, which does the same thing - # and must be consistent with this one. - if isinstance(obj, type): - if obj.__module__ == 'builtins': - return obj.__qualname__ - return f'{obj.__module__}.{obj.__qualname__}' - if obj is ...: - return '...' - if isinstance(obj, types.FunctionType): - return obj.__name__ if isinstance(obj, tuple): # Special case for `repr` of types with `ParamSpec`: return '[' + ', '.join(_type_repr(t) for t in obj) + ']' - return repr(obj) + return annotationlib.value_to_source(obj) def _collect_type_parameters(args, *, enforce_default_ordering: bool = True): @@ -2948,14 +2937,10 @@ def annotate(format): if format in (annotationlib.Format.VALUE, annotationlib.Format.FORWARDREF): return checked_types else: - return _convert_to_source(types) + return annotationlib.annotations_to_source(types) return annotate -def _convert_to_source(types): - return {n: t if isinstance(t, str) else _type_repr(t) for n, t in types.items()} - - # attributes prohibited to set in NamedTuple class syntax _prohibited = frozenset({'__new__', '__init__', '__slots__', '__getnewargs__', '_fields', '_field_defaults', @@ -3241,7 +3226,7 @@ def __annotate__(format): for n, tp in own.items() } elif format == annotationlib.Format.SOURCE: - own = _convert_to_source(own_annotations) + own = annotationlib.annotations_to_source(own_annotations) else: own = own_checked_annotations annos.update(own) From a3cb74c46aa00d4d552538bbde27cc9890f4157c Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 25 Sep 2024 14:52:37 -0700 Subject: [PATCH 3/4] more docs --- Doc/library/annotationlib.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index 3e03db101e1096..2219e37f6b0677 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -377,7 +377,9 @@ Functions This is meant as a helper for user-provided annotate functions that support the :attr:`~Format.SOURCE` format but - do not have access to the code creating the annotations. + do not have access to the code creating the annotations. It can also + be used to provide a user-friendly string representation for other + objects that contain values that are commonly encountered in annotations. .. versionadded:: 3.14 From 4ebc406865f757dda6fb7ed1af86478c0bfce39b Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 25 Sep 2024 14:53:27 -0700 Subject: [PATCH 4/4] format --- Lib/annotationlib.py | 11 +++++++---- Lib/test/test_annotationlib.py | 20 +++++++++++++++----- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index b465b1f3ab9245..a027f4de3dfed6 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -773,11 +773,11 @@ def value_to_source(value): """ if isinstance(value, type): - if value.__module__ == 'builtins': + if value.__module__ == "builtins": return value.__qualname__ - return f'{value.__module__}.{value.__qualname__}' + return f"{value.__module__}.{value.__qualname__}" if value is ...: - return '...' + return "..." if isinstance(value, (types.FunctionType, types.BuiltinFunctionType)): return value.__name__ return repr(value) @@ -785,7 +785,10 @@ def value_to_source(value): def annotations_to_source(annotations): """Convert an annotation dict containing values to approximately the SOURCE format.""" - return {n: t if isinstance(t, str) else value_to_source(t) for n, t in annotations.items()} + return { + n: t if isinstance(t, str) else value_to_source(t) + for n, t in annotations.items() + } def _get_and_call_annotate(obj, format): diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 180627e774b6da..dc1106aee1e2f1 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -7,7 +7,14 @@ import itertools import pickle import unittest -from annotationlib import Format, ForwardRef, get_annotations, get_annotate_function, annotations_to_source, value_to_source +from annotationlib import ( + Format, + ForwardRef, + get_annotations, + get_annotate_function, + annotations_to_source, + value_to_source, +) from typing import Unpack from test import support @@ -329,7 +336,10 @@ def test_name_lookup_without_eval(self): # namespaces without going through eval() self.assertIs(ForwardRef("int").evaluate(), int) self.assertIs(ForwardRef("int").evaluate(locals={"int": str}), str) - self.assertIs(ForwardRef("int").evaluate(locals={"int": float}, globals={"int": str}), float) + self.assertIs( + ForwardRef("int").evaluate(locals={"int": float}, globals={"int": str}), + float, + ) self.assertIs(ForwardRef("int").evaluate(globals={"int": str}), str) with support.swap_attr(builtins, "int", dict): self.assertIs(ForwardRef("int").evaluate(), dict) @@ -1100,11 +1110,11 @@ def test_annotations_to_source(self): self.assertEqual(annotations_to_source({}), {}) self.assertEqual(annotations_to_source({"x": int}), {"x": "int"}) self.assertEqual(annotations_to_source({"x": "int"}), {"x": "int"}) - self.assertEqual(annotations_to_source({"x": int, "y": str}), {"x": "int", "y": "str"}) + self.assertEqual( + annotations_to_source({"x": int, "y": str}), {"x": "int", "y": "str"} + ) class TestAnnotationLib(unittest.TestCase): def test__all__(self): support.check__all__(self, annotationlib) - -