From 271e387ba7f8dd558765510bf525a5bdf3bfd9ad Mon Sep 17 00:00:00 2001 From: Allan Daemon Date: Fri, 9 Oct 2020 02:04:46 -0300 Subject: [PATCH 1/8] Allows builtins to be generic aliases (PEP 585) --- mypy/main.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mypy/main.py b/mypy/main.py index 74758b40513e3..f6cf61a792e8f 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -81,6 +81,10 @@ def flush_errors(new_messages: List[str], serious: bool) -> None: f.write(msg + '\n') f.flush() + if options.python_version >= (3, 9): + from mypy.nodes import nongen_builtins + nongen_builtins.clear() + serious = False blockers = False res = None From 34726795a224528c9942cb8ba37258dd3efd49f1 Mon Sep 17 00:00:00 2001 From: Allan Daemon Date: Fri, 9 Oct 2020 02:04:46 -0300 Subject: [PATCH 2/8] fix test testNoSubcriptionOfStdlibCollections --- test-data/unit/pythoneval.test | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index e3eaf8a00ff3d..c401b57f8db99 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -854,6 +854,7 @@ _program.py:19: error: Dict entry 0 has incompatible type "str": "List[ _program.py:23: error: Invalid index type "str" for "MyDDict[Dict[_KT, _VT]]"; expected type "int" [case testNoSubcriptionOfStdlibCollections] +# flags: --python-version 3.6 import collections from collections import Counter from typing import TypeVar @@ -870,11 +871,11 @@ d[0] = 1 def f(d: collections.defaultdict[int, str]) -> None: ... [out] -_program.py:5: error: "defaultdict" is not subscriptable -_program.py:6: error: "Counter" is not subscriptable -_program.py:9: error: "defaultdict" is not subscriptable -_program.py:12: error: Invalid index type "int" for "defaultdict[str, int]"; expected type "str" -_program.py:14: error: "defaultdict" is not subscriptable, use "typing.DefaultDict" instead +_program.py:6: error: "defaultdict" is not subscriptable +_program.py:7: error: "Counter" is not subscriptable +_program.py:10: error: "defaultdict" is not subscriptable +_program.py:13: error: Invalid index type "int" for "defaultdict[str, int]"; expected type "str" +_program.py:15: error: "defaultdict" is not subscriptable, use "typing.DefaultDict" instead [case testCollectionsAliases] import typing as t From 1299956cf35573752cd19bbba2120d835e1b1aa7 Mon Sep 17 00:00:00 2001 From: Allan Daemon Date: Fri, 9 Oct 2020 02:04:46 -0300 Subject: [PATCH 3/8] fix test testDisallowAnyGenericsBuiltinCollections --- test-data/unit/cmdline.test | 1 + 1 file changed, 1 insertion(+) diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index 9d74bdc9a1bec..a72bf9704b447 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -645,6 +645,7 @@ a.__pow__() # E: All overload variants of "__pow__" of "int" require at least on # cmd: mypy m.py [file mypy.ini] \[mypy] +python_version=3.6 \[mypy-m] disallow_any_generics = True From ca5c841c2e0edf7bf15a8ff1b880f62068410982 Mon Sep 17 00:00:00 2001 From: Allan Daemon Date: Fri, 9 Oct 2020 02:04:46 -0300 Subject: [PATCH 4/8] Revert "Allows builtins to be generic aliases (PEP 585)" This reverts commit 22a4789d547121652c32de1ff25334ca6a9c3f0a. --- mypy/main.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index f6cf61a792e8f..74758b40513e3 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -81,10 +81,6 @@ def flush_errors(new_messages: List[str], serious: bool) -> None: f.write(msg + '\n') f.flush() - if options.python_version >= (3, 9): - from mypy.nodes import nongen_builtins - nongen_builtins.clear() - serious = False blockers = False res = None From dcfedebb72c752d6ede323da9dfd5089ca4d0612 Mon Sep 17 00:00:00 2001 From: Allan Daemon Date: Fri, 9 Oct 2020 02:04:46 -0300 Subject: [PATCH 5/8] create get_nongen_builtins() to access nongen_builtins depending on python version (for pep585) --- mypy/nodes.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index dff1dd6c1072d..f34f2daf95694 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -136,9 +136,13 @@ def get_column(self) -> int: 'builtins.frozenset': 'typing.FrozenSet', } # type: Final -nongen_builtins = {'builtins.tuple': 'typing.Tuple', +_nongen_builtins = {'builtins.tuple': 'typing.Tuple', 'builtins.enumerate': ''} # type: Final -nongen_builtins.update((name, alias) for alias, name in type_aliases.items()) +_nongen_builtins.update((name, alias) for alias, name in type_aliases.items()) + +def get_nongen_builtins(python_version): + # After 3.9 with pep585 generic builtins are allowed. + return _nongen_builtins if python_version < (3, 9) else {} RUNTIME_PROTOCOL_DECOS = ('typing.runtime_checkable', 'typing_extensions.runtime', From a9d099609722e677072a00d8843dd448b848d33e Mon Sep 17 00:00:00 2001 From: Allan Daemon Date: Fri, 9 Oct 2020 01:20:31 -0300 Subject: [PATCH 6/8] implements pep585 accepting generic builtins --- mypy/semanal.py | 11 ++++++----- mypy/typeanal.py | 30 +++++++++++++++++++----------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 2ea444bf4ab26..4c841a2bb260a 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -73,7 +73,7 @@ YieldExpr, ExecStmt, BackquoteExpr, ImportBase, AwaitExpr, IntExpr, FloatExpr, UnicodeExpr, TempNode, OverloadPart, PlaceholderNode, COVARIANT, CONTRAVARIANT, INVARIANT, - nongen_builtins, get_member_expr_fullname, REVEAL_TYPE, + get_nongen_builtins, get_member_expr_fullname, REVEAL_TYPE, REVEAL_LOCALS, is_final_node, TypedDictExpr, type_aliases_target_versions, EnumCallExpr, RUNTIME_PROTOCOL_DECOS, FakeExpression, Statement, AssignmentExpr, ParamSpecExpr @@ -446,7 +446,7 @@ def add_builtin_aliases(self, tree: MypyFile) -> None: target = self.named_type_or_none(target_name, []) assert target is not None # Transform List to List[Any], etc. - fix_instance_types(target, self.fail, self.note) + fix_instance_types(target, self.fail, self.note, self.options.python_version) alias_node = TypeAlias(target, alias, line=-1, column=-1, # there is no context no_args=True, normalized=True) @@ -2566,7 +2566,7 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool: # if the expected number of arguments is non-zero, so that aliases like A = List work. # However, eagerly expanding aliases like Text = str is a nice performance optimization. no_args = isinstance(res, Instance) and not res.args # type: ignore - fix_instance_types(res, self.fail, self.note) + fix_instance_types(res, self.fail, self.note, self.options.python_version) alias_node = TypeAlias(res, self.qualified_name(lvalue.name), s.line, s.column, alias_tvars=alias_tvars, no_args=no_args) if isinstance(s.rvalue, (IndexExpr, CallExpr)): # CallExpr is for `void = type(None)` @@ -3787,12 +3787,13 @@ def analyze_type_application(self, expr: IndexExpr) -> None: if isinstance(target, Instance): name = target.type.fullname if (alias.no_args and # this avoids bogus errors for already reported aliases - name in nongen_builtins and not alias.normalized): + name in get_nongen_builtins(self.options.python_version) and + not alias.normalized): self.fail(no_subscript_builtin_alias(name, propose_alt=False), expr) # ...or directly. else: n = self.lookup_type_node(base) - if n and n.fullname in nongen_builtins: + if n and n.fullname in get_nongen_builtins(self.options.python_version): self.fail(no_subscript_builtin_alias(n.fullname, propose_alt=False), expr) def analyze_type_application_args(self, expr: IndexExpr) -> Optional[List[Type]]: diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 7a7408d351e19..92a9bb962c137 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -21,7 +21,7 @@ from mypy.nodes import ( TypeInfo, Context, SymbolTableNode, Var, Expression, - nongen_builtins, check_arg_names, check_arg_kinds, ARG_POS, ARG_NAMED, + get_nongen_builtins, check_arg_names, check_arg_kinds, ARG_POS, ARG_NAMED, ARG_OPT, ARG_NAMED_OPT, ARG_STAR, ARG_STAR2, TypeVarExpr, TypeVarLikeExpr, ParamSpecExpr, TypeAlias, PlaceholderNode, SYMBOL_FUNCBASE_TYPES, Decorator, MypyFile ) @@ -94,6 +94,8 @@ def analyze_type_alias(node: Expression, def no_subscript_builtin_alias(name: str, propose_alt: bool = True) -> str: msg = '"{}" is not subscriptable'.format(name.split('.')[-1]) + # This should never be called if the python_version is 3.9 or newer + nongen_builtins = get_nongen_builtins((3, 8)) replacement = nongen_builtins[name] if replacement and propose_alt: msg += ', use "{}" instead'.format(replacement) @@ -194,7 +196,7 @@ def visit_unbound_type_nonoptional(self, t: UnboundType, defining_literal: bool) hook = self.plugin.get_type_analyze_hook(fullname) if hook is not None: return hook(AnalyzeTypeContext(t, t, self)) - if (fullname in nongen_builtins + if (fullname in get_nongen_builtins(self.options.python_version) and t.args and not self.allow_unnormalized and not self.api.is_future_flag_set("annotations")): @@ -241,6 +243,7 @@ def visit_unbound_type_nonoptional(self, t: UnboundType, defining_literal: bool) self.fail, self.note, disallow_any=disallow_any, + python_version=self.options.python_version, use_generic_error=True, unexpanded_type=t) return res @@ -342,7 +345,7 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Opt def get_omitted_any(self, typ: Type, fullname: Optional[str] = None) -> AnyType: disallow_any = not self.is_typeshed_stub and self.options.disallow_any_generics - return get_omitted_any(disallow_any, self.fail, self.note, typ, fullname) + return get_omitted_any(disallow_any, self.fail, self.note, typ, self.options.python_version, fullname) def analyze_type_with_type_info( self, info: TypeInfo, args: Sequence[Type], ctx: Context) -> Type: @@ -364,7 +367,8 @@ def analyze_type_with_type_info( if len(instance.args) != len(info.type_vars) and not self.defining_alias: fix_instance(instance, self.fail, self.note, disallow_any=self.options.disallow_any_generics and - not self.is_typeshed_stub) + not self.is_typeshed_stub, + python_version=self.options.python_version) tup = info.tuple_type if tup is not None: @@ -970,9 +974,11 @@ def tuple_type(self, items: List[Type]) -> TupleType: def get_omitted_any(disallow_any: bool, fail: MsgCallback, note: MsgCallback, - orig_type: Type, fullname: Optional[str] = None, + orig_type: Type, python_version: Tuple[int, int], + fullname: Optional[str] = None, unexpanded_type: Optional[Type] = None) -> AnyType: if disallow_any: + nongen_builtins = get_nongen_builtins(python_version) if fullname in nongen_builtins: typ = orig_type # We use a dedicated error message for builtin generics (as the most common case). @@ -1010,7 +1016,8 @@ def get_omitted_any(disallow_any: bool, fail: MsgCallback, note: MsgCallback, def fix_instance(t: Instance, fail: MsgCallback, note: MsgCallback, - disallow_any: bool, use_generic_error: bool = False, + disallow_any: bool, python_version: Tuple[int, int], + use_generic_error: bool = False, unexpanded_type: Optional[Type] = None,) -> None: """Fix a malformed instance by replacing all type arguments with Any. @@ -1021,7 +1028,7 @@ def fix_instance(t: Instance, fail: MsgCallback, note: MsgCallback, fullname = None # type: Optional[str] else: fullname = t.type.fullname - any_type = get_omitted_any(disallow_any, fail, note, t, fullname, unexpanded_type) + any_type = get_omitted_any(disallow_any, fail, note, t, python_version, fullname, unexpanded_type) t.args = (any_type,) * len(t.type.type_vars) return # Invalid number of type parameters. @@ -1280,21 +1287,22 @@ def make_optional_type(t: Type) -> Type: return UnionType([t, NoneType()], t.line, t.column) -def fix_instance_types(t: Type, fail: MsgCallback, note: MsgCallback) -> None: +def fix_instance_types(t: Type, fail: MsgCallback, note: MsgCallback, python_version: Tuple[int, int]) -> None: """Recursively fix all instance types (type argument count) in a given type. For example 'Union[Dict, List[str, int]]' will be transformed into 'Union[Dict[Any, Any], List[Any]]' in place. """ - t.accept(InstanceFixer(fail, note)) + t.accept(InstanceFixer(fail, note, python_version)) class InstanceFixer(TypeTraverserVisitor): - def __init__(self, fail: MsgCallback, note: MsgCallback) -> None: + def __init__(self, fail: MsgCallback, note: MsgCallback, python_version: Tuple[int, int]) -> None: self.fail = fail self.note = note + self.python_version = python_version def visit_instance(self, typ: Instance) -> None: super().visit_instance(typ) if len(typ.args) != len(typ.type.type_vars): - fix_instance(typ, self.fail, self.note, disallow_any=False, use_generic_error=True) + fix_instance(typ, self.fail, self.note, disallow_any=False, python_version=self.python_version, use_generic_error=True) From 40b5aed0cd204506e56f6dd8d2f851e9fbb79dc2 Mon Sep 17 00:00:00 2001 From: Allan Daemon Date: Wed, 14 Oct 2020 06:22:25 -0300 Subject: [PATCH 7/8] fix code style to mypy guidelines --- mypy/nodes.py | 6 ++++-- mypy/typeanal.py | 16 +++++++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index f34f2daf95694..fed1033a59464 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -137,13 +137,15 @@ def get_column(self) -> int: } # type: Final _nongen_builtins = {'builtins.tuple': 'typing.Tuple', - 'builtins.enumerate': ''} # type: Final + 'builtins.enumerate': ''} # type: Final _nongen_builtins.update((name, alias) for alias, name in type_aliases.items()) -def get_nongen_builtins(python_version): + +def get_nongen_builtins(python_version: Tuple[int, int]) -> Dict[str, str]: # After 3.9 with pep585 generic builtins are allowed. return _nongen_builtins if python_version < (3, 9) else {} + RUNTIME_PROTOCOL_DECOS = ('typing.runtime_checkable', 'typing_extensions.runtime', 'typing_extensions.runtime_checkable') # type: Final diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 92a9bb962c137..dfd1217cda8fb 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -345,7 +345,8 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Opt def get_omitted_any(self, typ: Type, fullname: Optional[str] = None) -> AnyType: disallow_any = not self.is_typeshed_stub and self.options.disallow_any_generics - return get_omitted_any(disallow_any, self.fail, self.note, typ, self.options.python_version, fullname) + return get_omitted_any(disallow_any, self.fail, self.note, typ, + self.options.python_version, fullname) def analyze_type_with_type_info( self, info: TypeInfo, args: Sequence[Type], ctx: Context) -> Type: @@ -1028,7 +1029,8 @@ def fix_instance(t: Instance, fail: MsgCallback, note: MsgCallback, fullname = None # type: Optional[str] else: fullname = t.type.fullname - any_type = get_omitted_any(disallow_any, fail, note, t, python_version, fullname, unexpanded_type) + any_type = get_omitted_any(disallow_any, fail, note, t, python_version, fullname, + unexpanded_type) t.args = (any_type,) * len(t.type.type_vars) return # Invalid number of type parameters. @@ -1287,7 +1289,8 @@ def make_optional_type(t: Type) -> Type: return UnionType([t, NoneType()], t.line, t.column) -def fix_instance_types(t: Type, fail: MsgCallback, note: MsgCallback, python_version: Tuple[int, int]) -> None: +def fix_instance_types(t: Type, fail: MsgCallback, note: MsgCallback, + python_version: Tuple[int, int]) -> None: """Recursively fix all instance types (type argument count) in a given type. For example 'Union[Dict, List[str, int]]' will be transformed into @@ -1297,7 +1300,9 @@ def fix_instance_types(t: Type, fail: MsgCallback, note: MsgCallback, python_ver class InstanceFixer(TypeTraverserVisitor): - def __init__(self, fail: MsgCallback, note: MsgCallback, python_version: Tuple[int, int]) -> None: + def __init__( + self, fail: MsgCallback, note: MsgCallback, python_version: Tuple[int, int] + ) -> None: self.fail = fail self.note = note self.python_version = python_version @@ -1305,4 +1310,5 @@ def __init__(self, fail: MsgCallback, note: MsgCallback, python_version: Tuple[i def visit_instance(self, typ: Instance) -> None: super().visit_instance(typ) if len(typ.args) != len(typ.type.type_vars): - fix_instance(typ, self.fail, self.note, disallow_any=False, python_version=self.python_version, use_generic_error=True) + fix_instance(typ, self.fail, self.note, disallow_any=False, + python_version=self.python_version, use_generic_error=True) From d4492aea4a2557269641fda6d8a5f94f9acf8500 Mon Sep 17 00:00:00 2001 From: cdce8p <30130371+cdce8p@users.noreply.github.com> Date: Sat, 17 Oct 2020 22:10:31 +0200 Subject: [PATCH 8/8] Added testcases for pep585ga --- mypy/test/testcheck.py | 1 + mypy/typeanal.py | 3 +- test-data/unit/check-generic-alias.test | 49 +++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 test-data/unit/check-generic-alias.test diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 34d9b66da0c14..a81ed113a7098 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -91,6 +91,7 @@ 'check-errorcodes.test', 'check-annotated.test', 'check-parameter-specification.test', + 'check-generic-alias.test', ] # Tests that use Python 3.8-only AST features (like expression-scoped ignores): diff --git a/mypy/typeanal.py b/mypy/typeanal.py index dfd1217cda8fb..adf8f35a1e7c5 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -308,7 +308,8 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Opt elif fullname == 'typing.Callable': return self.analyze_callable_type(t) elif (fullname == 'typing.Type' or - (fullname == 'builtins.type' and self.api.is_future_flag_set('annotations'))): + (fullname == 'builtins.type' and self.api.is_future_flag_set('annotations')) or + (fullname == 'builtins.type' and not self.options.python_version < (3, 9))): if len(t.args) == 0: if fullname == 'typing.Type': any_type = self.get_omitted_any(t) diff --git a/test-data/unit/check-generic-alias.test b/test-data/unit/check-generic-alias.test new file mode 100644 index 0000000000000..c0b5a81fdc207 --- /dev/null +++ b/test-data/unit/check-generic-alias.test @@ -0,0 +1,49 @@ +-- Test cases for generic aliases + +[case testGenericAliasImportCollections] +# flags: --python-version 3.9 +from collections import defaultdict, OrderedDict, ChainMap, Counter, deque + +t1: defaultdict[int, int] +t2: OrderedDict[int, int] +t3: ChainMap[int, int] +t4: Counter[int] +t5: deque[int] + +[builtins fixtures/tuple.pyi] + +[case testGenericAliasImportCollectionsABC] +# flags: --python-version 3.9 +from collections.abc import Awaitable, Coroutine, Iterable, Iterator, Mapping + +t1: Awaitable +t2: Coroutine +t3: Iterable +t4: Iterator +t5: Mapping + +[builtins fixtures/tuple.pyi] + +[case testGenericAliasImportBuiltIns] +# flags: --python-version 3.9 + +t1: type[int] + +[builtins fixtures/list.pyi] +[builtins fixtures/dict.pyi] + +[case testGenericAliasImportBuiltInsSet] +# flags: --python-version 3.9 + +t1: set[int] + +[builtins fixtures/set.pyi] + +[case testGenericAliasImportRe] +# flags: --python-version 3.9 +from re import Pattern, Match + +t1: Pattern[str] +t2: Match[str] + +[builtins fixtures/tuple.pyi]