From 6048a379311ed8ea5d002b25dee01a275634072a Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 11 Sep 2018 20:12:01 +0100 Subject: [PATCH 1/8] Re-organize subtype caches --- mypy/subtypes.py | 8 +++++--- mypy/typestate.py | 22 +++++++++++++++------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 3032aff9d061e..15c152df4458d 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -89,9 +89,11 @@ def is_subtype(left: Type, right: Type, ignore_promotions=ignore_promotions)) +def ignore_tvars(s: Type, t: Type, v: int) -> bool: + return True + + def is_subtype_ignoring_tvars(left: Type, right: Type) -> bool: - def ignore_tvars(s: Type, t: Type, v: int) -> bool: - return True return is_subtype(left, right, ignore_tvars) @@ -1033,7 +1035,7 @@ def __init__(self, right: Type, *, ignore_promotions: bool = False) -> None: @staticmethod def build_subtype_kind(*, ignore_promotions: bool = False) -> SubtypeKind: - return ('subtype_proper', ignore_promotions) + return ('subtype_proper', None, False, False, ignore_promotions) def _lookup_cache(self, left: Instance, right: Instance) -> bool: return TypeState.is_cached_subtype_check(self._subtype_kind, left, right) diff --git a/mypy/typestate.py b/mypy/typestate.py index 318c51cde21ce..4f596428f52e8 100644 --- a/mypy/typestate.py +++ b/mypy/typestate.py @@ -8,6 +8,7 @@ MYPY = False if MYPY: from typing import ClassVar + from mypy.subtypes import TypeParameterChecker from mypy.nodes import TypeInfo from mypy.types import Instance from mypy.server.trigger import make_trigger @@ -17,11 +18,11 @@ # A tuple encoding the specific conditions under which we performed the subtype check. # (e.g. did we want a proper subtype? A regular subtype while ignoring variance?) -SubtypeKind = Tuple[Any, ...] +SubtypeKind = Tuple[str, Optional['TypeParameterChecker'], bool, bool, bool] # A cache that keeps track of whether the given TypeInfo is a part of a particular # subtype relationship -SubtypeCache = Dict[TypeInfo, Dict[SubtypeKind, Set[SubtypeRelationship]]] +SubtypeCache = Dict[SubtypeKind, Dict[TypeInfo, Set[SubtypeRelationship]]] class TypeState: @@ -83,7 +84,9 @@ def reset_all_subtype_caches(cls) -> None: @classmethod def reset_subtype_caches_for(cls, info: TypeInfo) -> None: """Reset subtype caches (if any) for a given supertype TypeInfo.""" - cls._subtype_caches.setdefault(info, dict()).clear() + for cache in cls._subtype_caches.values(): + if info in cache: + cache[info].clear() @classmethod def reset_all_subtype_caches_for(cls, info: TypeInfo) -> None: @@ -93,14 +96,19 @@ def reset_all_subtype_caches_for(cls, info: TypeInfo) -> None: @classmethod def is_cached_subtype_check(cls, kind: SubtypeKind, left: Instance, right: Instance) -> bool: - subtype_kinds = cls._subtype_caches.setdefault(right.type, dict()) - return (left, right) in subtype_kinds.setdefault(kind, set()) + if kind not in cls._subtype_caches: + return False + cache = cls._subtype_caches[kind] + info = right.type + if info not in cache: + return False + return (left, right) in cache[info] @classmethod def record_subtype_cache_entry(cls, kind: SubtypeKind, left: Instance, right: Instance) -> None: - subtype_kinds = cls._subtype_caches.setdefault(right.type, dict()) - subtype_kinds.setdefault(kind, set()).add((left, right)) + cache = cls._subtype_caches.setdefault(kind, dict()) + cache.setdefault(right.type, set()).add((left, right)) @classmethod def reset_protocol_deps(cls) -> None: From 1a799d0966a037bd807d2c93d4dd61cbfbaadb1f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 11 Sep 2018 20:19:02 +0100 Subject: [PATCH 2/8] Avoid import cycle --- mypy/subtypes.py | 5 +---- mypy/typestate.py | 9 +++++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 15c152df4458d..5f9ea891c8415 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -20,7 +20,7 @@ from mypy.maptype import map_instance_to_supertype from mypy.expandtype import expand_type_by_instance from mypy.sametypes import is_same_type -from mypy.typestate import TypeState, SubtypeKind +from mypy.typestate import TypeState, SubtypeKind, TypeParameterChecker from mypy import experiments @@ -31,9 +31,6 @@ IS_CLASS_OR_STATIC = 3 -TypeParameterChecker = Callable[[Type, Type, int], bool] - - def check_type_parameter(lefta: Type, righta: Type, variance: int) -> bool: if variance == COVARIANT: return is_subtype(lefta, righta, check_type_parameter) diff --git a/mypy/typestate.py b/mypy/typestate.py index 4f596428f52e8..03f397fdf258b 100644 --- a/mypy/typestate.py +++ b/mypy/typestate.py @@ -3,22 +3,23 @@ and potentially other mutable TypeInfo state. This module contains mutable global state. """ -from typing import Any, Dict, Set, Tuple, Optional +from typing import Any, Dict, Set, Tuple, Optional, Callable MYPY = False if MYPY: from typing import ClassVar - from mypy.subtypes import TypeParameterChecker from mypy.nodes import TypeInfo -from mypy.types import Instance +from mypy.types import Instance, Type from mypy.server.trigger import make_trigger # Represents that the 'left' instance is a subtype of the 'right' instance SubtypeRelationship = Tuple[Instance, Instance] +TypeParameterChecker = Callable[[Type, Type, int], bool] + # A tuple encoding the specific conditions under which we performed the subtype check. # (e.g. did we want a proper subtype? A regular subtype while ignoring variance?) -SubtypeKind = Tuple[str, Optional['TypeParameterChecker'], bool, bool, bool] +SubtypeKind = Tuple[str, Optional[TypeParameterChecker], bool, bool, bool] # A cache that keeps track of whether the given TypeInfo is a part of a particular # subtype relationship From 301a89b69cf07788ad34d6ded1c9352d46c2eca3 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 11 Sep 2018 20:34:02 +0100 Subject: [PATCH 3/8] Use heterogeneous tuples as keys --- mypy/subtypes.py | 6 ++++-- mypy/typestate.py | 8 +++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 5f9ea891c8415..5753343b1d80e 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -20,7 +20,7 @@ from mypy.maptype import map_instance_to_supertype from mypy.expandtype import expand_type_by_instance from mypy.sametypes import is_same_type -from mypy.typestate import TypeState, SubtypeKind, TypeParameterChecker +from mypy.typestate import TypeState, SubtypeKind from mypy import experiments @@ -30,6 +30,8 @@ IS_CLASSVAR = 2 IS_CLASS_OR_STATIC = 3 +TypeParameterChecker = Callable[[Type, Type, int], bool] + def check_type_parameter(lefta: Type, righta: Type, variance: int) -> bool: if variance == COVARIANT: @@ -1032,7 +1034,7 @@ def __init__(self, right: Type, *, ignore_promotions: bool = False) -> None: @staticmethod def build_subtype_kind(*, ignore_promotions: bool = False) -> SubtypeKind: - return ('subtype_proper', None, False, False, ignore_promotions) + return ('subtype_proper', ignore_promotions) def _lookup_cache(self, left: Instance, right: Instance) -> bool: return TypeState.is_cached_subtype_check(self._subtype_kind, left, right) diff --git a/mypy/typestate.py b/mypy/typestate.py index 03f397fdf258b..16a40aebf0cbb 100644 --- a/mypy/typestate.py +++ b/mypy/typestate.py @@ -3,23 +3,21 @@ and potentially other mutable TypeInfo state. This module contains mutable global state. """ -from typing import Any, Dict, Set, Tuple, Optional, Callable +from typing import Any, Dict, Set, Tuple, Optional MYPY = False if MYPY: from typing import ClassVar from mypy.nodes import TypeInfo -from mypy.types import Instance, Type +from mypy.types import Instance from mypy.server.trigger import make_trigger # Represents that the 'left' instance is a subtype of the 'right' instance SubtypeRelationship = Tuple[Instance, Instance] -TypeParameterChecker = Callable[[Type, Type, int], bool] - # A tuple encoding the specific conditions under which we performed the subtype check. # (e.g. did we want a proper subtype? A regular subtype while ignoring variance?) -SubtypeKind = Tuple[str, Optional[TypeParameterChecker], bool, bool, bool] +SubtypeKind = Tuple[Any, ...] # A cache that keeps track of whether the given TypeInfo is a part of a particular # subtype relationship From 818b9bc61483960c1b4d2dbab467549eb973b319 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 11 Sep 2018 21:29:28 +0100 Subject: [PATCH 4/8] Refactor subtype cache kinds to use only boolean flags --- mypy/subtypes.py | 64 ++++++++++++++++++++++++++--------------------- mypy/typestate.py | 2 +- 2 files changed, 36 insertions(+), 30 deletions(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 5753343b1d80e..c6b078089a31e 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -35,16 +35,21 @@ def check_type_parameter(lefta: Type, righta: Type, variance: int) -> bool: if variance == COVARIANT: - return is_subtype(lefta, righta, check_type_parameter) + return is_subtype(lefta, righta) elif variance == CONTRAVARIANT: - return is_subtype(righta, lefta, check_type_parameter) + return is_subtype(righta, lefta) else: - return is_equivalent(lefta, righta, check_type_parameter) + return is_equivalent(lefta, righta) + + +def ignore_type_parameter(s: Type, t: Type, v: int) -> bool: + return True def is_subtype(left: Type, right: Type, - type_parameter_checker: Optional[TypeParameterChecker] = None, - *, ignore_pos_arg_names: bool = False, + *, + ignore_type_params: bool = False, + ignore_pos_arg_names: bool = False, ignore_declared_variance: bool = False, ignore_promotions: bool = False) -> bool: """Is 'left' subtype of 'right'? @@ -58,7 +63,6 @@ def is_subtype(left: Type, right: Type, between the type arguments (e.g., A and B), taking the variance of the type var into account. """ - type_parameter_checker = type_parameter_checker or check_type_parameter if (isinstance(right, AnyType) or isinstance(right, UnboundType) or isinstance(right, ErasedType)): return True @@ -66,7 +70,8 @@ def is_subtype(left: Type, right: Type, # Normally, when 'left' is not itself a union, the only way # 'left' can be a subtype of the union 'right' is if it is a # subtype of one of the items making up the union. - is_subtype_of_item = any(is_subtype(left, item, type_parameter_checker, + is_subtype_of_item = any(is_subtype(left, item, + ignore_type_params=ignore_type_params, ignore_pos_arg_names=ignore_pos_arg_names, ignore_declared_variance=ignore_declared_variance, ignore_promotions=ignore_promotions) @@ -82,58 +87,58 @@ def is_subtype(left: Type, right: Type, elif is_subtype_of_item: return True # otherwise, fall through - return left.accept(SubtypeVisitor(right, type_parameter_checker, + return left.accept(SubtypeVisitor(right, + ignore_type_params=ignore_type_params, ignore_pos_arg_names=ignore_pos_arg_names, ignore_declared_variance=ignore_declared_variance, ignore_promotions=ignore_promotions)) -def ignore_tvars(s: Type, t: Type, v: int) -> bool: - return True - - def is_subtype_ignoring_tvars(left: Type, right: Type) -> bool: - return is_subtype(left, right, ignore_tvars) + return is_subtype(left, right, ignore_type_params=True) -def is_equivalent(a: Type, - b: Type, - type_parameter_checker: Optional[TypeParameterChecker] = None, +def is_equivalent(a: Type, b: Type, *, + ignore_type_params: bool = False, ignore_pos_arg_names: bool = False ) -> bool: return ( - is_subtype(a, b, type_parameter_checker, ignore_pos_arg_names=ignore_pos_arg_names) - and is_subtype(b, a, type_parameter_checker, ignore_pos_arg_names=ignore_pos_arg_names)) + is_subtype(a, b, ignore_type_params=ignore_type_params, + ignore_pos_arg_names=ignore_pos_arg_names) + and is_subtype(b, a, ignore_type_params=ignore_type_params, + ignore_pos_arg_names=ignore_pos_arg_names)) class SubtypeVisitor(TypeVisitor[bool]): def __init__(self, right: Type, - type_parameter_checker: TypeParameterChecker, - *, ignore_pos_arg_names: bool = False, + *, + ignore_type_params: bool, + ignore_pos_arg_names: bool = False, ignore_declared_variance: bool = False, ignore_promotions: bool = False) -> None: self.right = right - self.check_type_parameter = type_parameter_checker + self.ignore_type_params = ignore_type_params self.ignore_pos_arg_names = ignore_pos_arg_names self.ignore_declared_variance = ignore_declared_variance self.ignore_promotions = ignore_promotions + self.check_type_parameter = (ignore_type_parameter if ignore_type_params else + check_type_parameter) self._subtype_kind = SubtypeVisitor.build_subtype_kind( - type_parameter_checker=type_parameter_checker, + ignore_type_params=ignore_type_params, ignore_pos_arg_names=ignore_pos_arg_names, ignore_declared_variance=ignore_declared_variance, ignore_promotions=ignore_promotions) @staticmethod def build_subtype_kind(*, - type_parameter_checker: Optional[TypeParameterChecker] = None, + ignore_type_params: bool = False, ignore_pos_arg_names: bool = False, ignore_declared_variance: bool = False, ignore_promotions: bool = False) -> SubtypeKind: - type_parameter_checker = type_parameter_checker or check_type_parameter - return ('subtype', - type_parameter_checker, + return (False, # is proper subtype? + ignore_type_params, ignore_pos_arg_names, ignore_declared_variance, ignore_promotions) @@ -146,7 +151,7 @@ def _record_cache(self, left: Instance, right: Instance) -> None: def _is_subtype(self, left: Type, right: Type) -> bool: return is_subtype(left, right, - type_parameter_checker=self.check_type_parameter, + ignore_type_params=self.ignore_type_params, ignore_pos_arg_names=self.ignore_pos_arg_names, ignore_declared_variance=self.ignore_declared_variance, ignore_promotions=self.ignore_promotions) @@ -304,7 +309,8 @@ def visit_typeddict_type(self, left: TypedDictType) -> bool: if not left.names_are_wider_than(right): return False for name, l, r in left.zip(right): - if not is_equivalent(l, r, self.check_type_parameter): + if not is_equivalent(l, r, + ignore_type_params=self.ignore_type_params): return False # Non-required key is not compatible with a required key since # indexing may fail unexpectedly if a required key is missing. @@ -1034,7 +1040,7 @@ def __init__(self, right: Type, *, ignore_promotions: bool = False) -> None: @staticmethod def build_subtype_kind(*, ignore_promotions: bool = False) -> SubtypeKind: - return ('subtype_proper', ignore_promotions) + return (True, ignore_promotions) def _lookup_cache(self, left: Instance, right: Instance) -> bool: return TypeState.is_cached_subtype_check(self._subtype_kind, left, right) diff --git a/mypy/typestate.py b/mypy/typestate.py index 16a40aebf0cbb..1b328a56b711d 100644 --- a/mypy/typestate.py +++ b/mypy/typestate.py @@ -17,7 +17,7 @@ # A tuple encoding the specific conditions under which we performed the subtype check. # (e.g. did we want a proper subtype? A regular subtype while ignoring variance?) -SubtypeKind = Tuple[Any, ...] +SubtypeKind = Tuple[bool, ...] # A cache that keeps track of whether the given TypeInfo is a part of a particular # subtype relationship From a58cab396f478e1c2124e76a5c40e86935b64193 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 11 Sep 2018 21:39:34 +0100 Subject: [PATCH 5/8] Remove one-liners --- mypy/subtypes.py | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index c6b078089a31e..66201f5ec6134 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -143,12 +143,6 @@ def build_subtype_kind(*, ignore_declared_variance, ignore_promotions) - def _lookup_cache(self, left: Instance, right: Instance) -> bool: - return TypeState.is_cached_subtype_check(self._subtype_kind, left, right) - - def _record_cache(self, left: Instance, right: Instance) -> None: - TypeState.record_subtype_cache_entry(self._subtype_kind, left, right) - def _is_subtype(self, left: Type, right: Type) -> bool: return is_subtype(left, right, ignore_type_params=self.ignore_type_params, @@ -196,12 +190,12 @@ def visit_instance(self, left: Instance) -> bool: if isinstance(right, TupleType) and right.fallback.type.is_enum: return self._is_subtype(left, right.fallback) if isinstance(right, Instance): - if self._lookup_cache(left, right): + if TypeState.is_cached_subtype_check(self._subtype_kind, left, right): return True if not self.ignore_promotions: for base in left.type.mro: if base._promote and self._is_subtype(base._promote, self.right): - self._record_cache(left, right) + TypeState.record_subtype_cache_entry(self._subtype_kind, left, right) return True rname = right.type.fullname() # Always try a nominal check if possible, @@ -214,7 +208,7 @@ def visit_instance(self, left: Instance) -> bool: for lefta, righta, tvar in zip(t.args, right.args, right.type.defn.type_vars)) if nominal: - self._record_cache(left, right) + TypeState.record_subtype_cache_entry(self._subtype_kind, left, right) return nominal if right.type.is_protocol and is_protocol_implementation(left, right): return True @@ -1042,12 +1036,6 @@ def __init__(self, right: Type, *, ignore_promotions: bool = False) -> None: def build_subtype_kind(*, ignore_promotions: bool = False) -> SubtypeKind: return (True, ignore_promotions) - def _lookup_cache(self, left: Instance, right: Instance) -> bool: - return TypeState.is_cached_subtype_check(self._subtype_kind, left, right) - - def _record_cache(self, left: Instance, right: Instance) -> None: - TypeState.record_subtype_cache_entry(self._subtype_kind, left, right) - def _is_proper_subtype(self, left: Type, right: Type) -> bool: return is_proper_subtype(left, right, ignore_promotions=self.ignore_promotions) @@ -1080,12 +1068,12 @@ def visit_deleted_type(self, left: DeletedType) -> bool: def visit_instance(self, left: Instance) -> bool: right = self.right if isinstance(right, Instance): - if self._lookup_cache(left, right): + if TypeState.is_cached_subtype_check(self._subtype_kind, left, right): return True if not self.ignore_promotions: for base in left.type.mro: if base._promote and self._is_proper_subtype(base._promote, right): - self._record_cache(left, right) + TypeState.record_subtype_cache_entry(self._subtype_kind, left, right) return True if left.type.has_base(right.type.fullname()): @@ -1102,7 +1090,7 @@ def check_argument(leftarg: Type, rightarg: Type, variance: int) -> bool: nominal = all(check_argument(ta, ra, tvar.variance) for ta, ra, tvar in zip(left.args, right.args, right.type.defn.type_vars)) if nominal: - self._record_cache(left, right) + TypeState.record_subtype_cache_entry(self._subtype_kind, left, right) return nominal if (right.type.is_protocol and is_protocol_implementation(left, right, proper_subtype=True)): From 5704437ff8a4009b003de8d8f3c169002c784187 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 11 Sep 2018 21:57:33 +0100 Subject: [PATCH 6/8] Switch back the order of caches --- mypy/typestate.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/mypy/typestate.py b/mypy/typestate.py index 1b328a56b711d..d694124a24fa2 100644 --- a/mypy/typestate.py +++ b/mypy/typestate.py @@ -21,7 +21,7 @@ # A cache that keeps track of whether the given TypeInfo is a part of a particular # subtype relationship -SubtypeCache = Dict[SubtypeKind, Dict[TypeInfo, Set[SubtypeRelationship]]] +SubtypeCache = Dict[TypeInfo, Dict[SubtypeKind, Set[SubtypeRelationship]]] class TypeState: @@ -83,9 +83,8 @@ def reset_all_subtype_caches(cls) -> None: @classmethod def reset_subtype_caches_for(cls, info: TypeInfo) -> None: """Reset subtype caches (if any) for a given supertype TypeInfo.""" - for cache in cls._subtype_caches.values(): - if info in cache: - cache[info].clear() + if info in cls._subtype_caches: + cls._subtype_caches[info].clear() @classmethod def reset_all_subtype_caches_for(cls, info: TypeInfo) -> None: @@ -95,19 +94,19 @@ def reset_all_subtype_caches_for(cls, info: TypeInfo) -> None: @classmethod def is_cached_subtype_check(cls, kind: SubtypeKind, left: Instance, right: Instance) -> bool: - if kind not in cls._subtype_caches: - return False - cache = cls._subtype_caches[kind] info = right.type - if info not in cache: + if info not in cls._subtype_caches: + return False + cache = cls._subtype_caches[info] + if kind not in cache: return False - return (left, right) in cache[info] + return (left, right) in cache[kind] @classmethod def record_subtype_cache_entry(cls, kind: SubtypeKind, left: Instance, right: Instance) -> None: - cache = cls._subtype_caches.setdefault(kind, dict()) - cache.setdefault(right.type, set()).add((left, right)) + cache = cls._subtype_caches.setdefault(right.type, dict()) + cache.setdefault(kind, set()).add((left, right)) @classmethod def reset_protocol_deps(cls) -> None: From 4c5dd0328e43e8f4175396d65fa4031e1f4830be Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 12 Sep 2018 00:34:54 +0100 Subject: [PATCH 7/8] Fix whitespace --- mypy/subtypes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 66201f5ec6134..47ef0202d8641 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -30,6 +30,7 @@ IS_CLASSVAR = 2 IS_CLASS_OR_STATIC = 3 + TypeParameterChecker = Callable[[Type, Type, int], bool] From a2e772d3a01fda06ad69f3e4060234bfc62f751e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 12 Sep 2018 00:49:24 +0100 Subject: [PATCH 8/8] Fix lint --- mypy/subtypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 47ef0202d8641..b40806917644d 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -125,7 +125,7 @@ def __init__(self, right: Type, self.ignore_declared_variance = ignore_declared_variance self.ignore_promotions = ignore_promotions self.check_type_parameter = (ignore_type_parameter if ignore_type_params else - check_type_parameter) + check_type_parameter) self._subtype_kind = SubtypeVisitor.build_subtype_kind( ignore_type_params=ignore_type_params, ignore_pos_arg_names=ignore_pos_arg_names,