Skip to content

Commit

Permalink
Add partial overload checks (#5476)
Browse files Browse the repository at this point in the history 
This pull request adds more robust support for detecting partially
overlapping types. Specifically, it detects overlaps with...

1. TypedDicts
2. Tuples
3. Unions
4. TypeVars
5. Generic types containing variations of the above.

This new overlapping type check is used for detecting unsafe overlaps
with overloads and operator methods as well as for performing 
reachability/unreachability analysis in `isinstance` and `if x is None`
checks and the like.

This PR also removes some (now unused) code that used to be used
for detecting overlaps with operators.

This pull request builds on top of #5474
and #5475 and supersedes 
#5475.
  • Loading branch information
@Michael0x2a
Michael0x2a authored Aug 27, 2018
1 parent 4e6d753 commit dd70710
Show file tree
Hide file tree
Showing 8 changed files with 1,050 additions and 264 deletions.
225 changes: 107 additions & 118 deletions mypy/checker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from contextlib import contextmanager

from typing import (
Dict, Set, List, cast, Tuple, TypeVar, Union, Optional, NamedTuple, Iterator, Any
Dict, Set, List, cast, Tuple, TypeVar, Union, Optional, NamedTuple, Iterator, Iterable, Any
)

from mypy.errors import Errors, report_internal_error
Expand All @@ -30,7 +30,7 @@
Type, AnyType, CallableType, FunctionLike, Overloaded, TupleType, TypedDictType,
Instance, NoneTyp, strip_type, TypeType, TypeOfAny,
UnionType, TypeVarId, TypeVarType, PartialType, DeletedType, UninhabitedType, TypeVarDef,
true_only, false_only, function_type, is_named_instance, union_items,
true_only, false_only, function_type, is_named_instance, union_items, TypeQuery
)
from mypy.sametypes import is_same_type, is_same_types
from mypy.messages import MessageBuilder, make_inferred_type_note
Expand All @@ -55,7 +55,7 @@
from mypy.join import join_types
from mypy.treetransform import TransformVisitor
from mypy.binder import ConditionalTypeBinder, get_declaration
from mypy.meet import is_overlapping_types, is_partially_overlapping_types
from mypy.meet import is_overlapping_erased_types, is_overlapping_types
from mypy.options import Options
from mypy.plugin import Plugin, CheckerPluginInterface
from mypy.sharedparse import BINARY_MAGIC_METHODS
Expand Down Expand Up @@ -495,12 +495,12 @@ def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None:

# Is the overload alternative's arguments subtypes of the implementation's?
if not is_callable_compatible(impl, sig1,
is_compat=is_subtype,
is_compat=is_subtype_no_promote,
ignore_return=True):
self.msg.overloaded_signatures_arg_specific(i + 1, defn.impl)

# Is the overload alternative's return type a subtype of the implementation's?
if not is_subtype(sig1.ret_type, impl.ret_type):
if not is_subtype_no_promote(sig1.ret_type, impl.ret_type):
self.msg.overloaded_signatures_ret_specific(i + 1, defn.impl)

# Here's the scoop about generators and coroutines.
Expand Down Expand Up @@ -3156,7 +3156,7 @@ def find_isinstance_check(self, node: Expression
else:
optional_type, comp_type = second_type, first_type
optional_expr = node.operands[1]
if is_overlapping_types(optional_type, comp_type):
if is_overlapping_erased_types(optional_type, comp_type):
return {optional_expr: remove_optional(optional_type)}, {}
elif node.operators in [['in'], ['not in']]:
expr = node.operands[0]
Expand All @@ -3167,7 +3167,7 @@ def find_isinstance_check(self, node: Expression
right_type.type.fullname() != 'builtins.object'))
if (right_type and right_ok and is_optional(left_type) and
literal(expr) == LITERAL_TYPE and not is_literal_none(expr) and
is_overlapping_types(left_type, right_type)):
is_overlapping_erased_types(left_type, right_type)):
if node.operators == ['in']:
return {expr: remove_optional(left_type)}, {}
if node.operators == ['not in']:
Expand Down Expand Up @@ -3515,7 +3515,8 @@ def conditional_type_map(expr: Expression,
and is_proper_subtype(current_type, proposed_type)):
# Expression is always of one of the types in proposed_type_ranges
return {}, None
elif not is_overlapping_types(current_type, proposed_type):
elif not is_overlapping_types(current_type, proposed_type,
prohibit_none_typevar_overlap=True):
# Expression is never of any type in proposed_type_ranges
return None, {}
else:
Expand Down Expand Up @@ -3731,9 +3732,9 @@ def are_argument_counts_overlapping(t: CallableType, s: CallableType) -> bool:

def is_unsafe_overlapping_overload_signatures(signature: CallableType,
other: CallableType) -> bool:
"""Check if two overloaded function signatures may be unsafely overlapping.
"""Check if two overloaded signatures are unsafely overlapping or partially overlapping.
We consider two functions 's' and 't' to be unsafely overlapping both if
We consider two functions 's' and 't' to be unsafely overlapping if both
of the following are true:
1. s's parameters are all more precise or partially overlapping with t's
Expand All @@ -3742,26 +3743,98 @@ def is_unsafe_overlapping_overload_signatures(signature: CallableType,
Assumes that 'signature' appears earlier in the list of overload
alternatives then 'other' and that their argument counts are overlapping.
"""
# TODO: Handle partially overlapping parameter types
# Try detaching callables from the containing class so that all TypeVars
# are treated as being free.
#
# For example, the signatures "f(x: Union[A, B]) -> int" and "f(x: Union[B, C]) -> str"
# is unsafe: the parameter types are partially overlapping.
# This lets us identify cases where the two signatures use completely
# incompatible types -- e.g. see the testOverloadingInferUnionReturnWithMixedTypevars
# test case.
signature = detach_callable(signature)
other = detach_callable(other)

# Note: We repeat this check twice in both directions due to a slight
# asymmetry in 'is_callable_compatible'. When checking for partial overlaps,
# we attempt to unify 'signature' and 'other' both against each other.
#
# To fix this, we need to either modify meet.is_overlapping_types or add a new
# function and use "is_more_precise(...) or is_partially_overlapping(...)" for the is_compat
# checks.
# If 'signature' cannot be unified with 'other', we end early. However,
# if 'other' cannot be modified with 'signature', the function continues
# using the older version of 'other'.
#
# (We already have a rudimentary implementation of 'is_partially_overlapping', but it only
# attempts to handle the obvious cases -- see its docstring for more info.)

def is_more_precise_or_partially_overlapping(t: Type, s: Type) -> bool:
return is_more_precise(t, s) or is_partially_overlapping_types(t, s)

return is_callable_compatible(signature, other,
is_compat=is_more_precise_or_partially_overlapping,
is_compat_return=lambda l, r: not is_subtype(l, r),
# This discrepancy is unfortunately difficult to get rid of, so we repeat the
# checks twice in both directions for now.
return (is_callable_compatible(signature, other,
is_compat=is_overlapping_types_no_promote,
is_compat_return=lambda l, r: not is_subtype_no_promote(l, r),
ignore_return=False,
check_args_covariantly=True,
allow_partial_overlap=True)
allow_partial_overlap=True) or
is_callable_compatible(other, signature,
is_compat=is_overlapping_types_no_promote,
is_compat_return=lambda l, r: not is_subtype_no_promote(r, l),
ignore_return=False,
check_args_covariantly=False,
allow_partial_overlap=True))


def detach_callable(typ: CallableType) -> CallableType:
"""Ensures that the callable's type variables are 'detached' and independent of the context.
A callable normally keeps track of the type variables it uses within its 'variables' field.
However, if the callable is from a method and that method is using a class type variable,
the callable will not keep track of that type variable since it belongs to the class.
This function will traverse the callable and find all used type vars and add them to the
variables field if it isn't already present.
The caller can then unify on all type variables whether or not the callable is originally
from a class or not."""
type_list = typ.arg_types + [typ.ret_type]

appear_map = {} # type: Dict[str, List[int]]
for i, inner_type in enumerate(type_list):
typevars_available = inner_type.accept(TypeVarExtractor())
for var in typevars_available:
if var.fullname not in appear_map:
appear_map[var.fullname] = []
appear_map[var.fullname].append(i)

used_type_var_names = set()
for var_name, appearances in appear_map.items():
used_type_var_names.add(var_name)

all_type_vars = typ.accept(TypeVarExtractor())
new_variables = []
for var in set(all_type_vars):
if var.fullname not in used_type_var_names:
continue
new_variables.append(TypeVarDef(
name=var.name,
fullname=var.fullname,
id=var.id,
values=var.values,
upper_bound=var.upper_bound,
variance=var.variance,
))
out = typ.copy_modified(
variables=new_variables,
arg_types=type_list[:-1],
ret_type=type_list[-1],
)
return out


class TypeVarExtractor(TypeQuery[List[TypeVarType]]):
def __init__(self) -> None:
super().__init__(self._merge)

def _merge(self, iter: Iterable[List[TypeVarType]]) -> List[TypeVarType]:
out = []
for item in iter:
out.extend(item)
return out

def visit_type_var(self, t: TypeVarType) -> List[TypeVarType]:
return [t]


def overload_can_never_match(signature: CallableType, other: CallableType) -> bool:
Expand All @@ -3787,69 +3860,6 @@ def overload_can_never_match(signature: CallableType, other: CallableType) -> bo
ignore_return=True)


def is_unsafe_overlapping_operator_signatures(signature: Type, other: Type) -> bool:
"""Check if two operator method signatures may be unsafely overlapping.
Two signatures s and t are overlapping if both can be valid for the same
statically typed values and the return types are incompatible.
Assume calls are first checked against 'signature', then against 'other'.
Thus if 'signature' is more general than 'other', there is no unsafe
overlapping.
TODO: Clean up this function and make it not perform type erasure.
Context: This function was previously used to make sure both overloaded
functions and operator methods were not unsafely overlapping.
We changed the semantics for we should handle overloaded definitions,
but not operator functions. (We can't reuse the same semantics for both:
the overload semantics are too restrictive here).
We should rewrite this method so that:
1. It uses many of the improvements made to overloads: in particular,
eliminating type erasure.
2. It contains just the logic necessary for operator methods.
"""
if isinstance(signature, CallableType):
if isinstance(other, CallableType):
# TODO varargs
# TODO keyword args
# TODO erasure
# TODO allow to vary covariantly
# Check if the argument counts are overlapping.
min_args = max(signature.min_args, other.min_args)
max_args = min(len(signature.arg_types), len(other.arg_types))
if min_args > max_args:
# Argument counts are not overlapping.
return False
# Signatures are overlapping iff if they are overlapping for the
# smallest common argument count.
for i in range(min_args):
t1 = signature.arg_types[i]
t2 = other.arg_types[i]
if not is_overlapping_types(t1, t2):
return False
# All arguments types for the smallest common argument count are
# overlapping => the signature is overlapping. The overlapping is
# safe if the return types are identical.
if is_same_type(signature.ret_type, other.ret_type):
return False
# If the first signature has more general argument types, the
# latter will never be called
if is_more_general_arg_prefix(signature, other):
return False
# Special case: all args are subtypes, and returns are subtypes
if (all(is_proper_subtype(s, o)
for (s, o) in zip(signature.arg_types, other.arg_types)) and
is_subtype(signature.ret_type, other.ret_type)):
return False
return not is_more_precise_signature(signature, other)
return True


def is_more_general_arg_prefix(t: FunctionLike, s: FunctionLike) -> bool:
"""Does t have wider arguments than s?"""
# TODO should an overload with additional items be allowed to be more
Expand All @@ -3867,20 +3877,6 @@ def is_more_general_arg_prefix(t: FunctionLike, s: FunctionLike) -> bool:
return False


def is_equivalent_type_var_def(tv1: TypeVarDef, tv2: TypeVarDef) -> bool:
"""Are type variable definitions equivalent?
Ignore ids, locations in source file and names.
"""
return (
tv1.variance == tv2.variance
and is_same_types(tv1.values, tv2.values)
and ((tv1.upper_bound is None and tv2.upper_bound is None)
or (tv1.upper_bound is not None
and tv2.upper_bound is not None
and is_same_type(tv1.upper_bound, tv2.upper_bound))))


def is_same_arg_prefix(t: CallableType, s: CallableType) -> bool:
return is_callable_compatible(t, s,
is_compat=is_same_type,
Expand All @@ -3889,21 +3885,6 @@ def is_same_arg_prefix(t: CallableType, s: CallableType) -> bool:
ignore_pos_arg_names=True)


def is_more_precise_signature(t: CallableType, s: CallableType) -> bool:
"""Is t more precise than s?
A signature t is more precise than s if all argument types and the return
type of t are more precise than the corresponding types in s.
Assume that the argument kinds and names are compatible, and that the
argument counts are overlapping.
"""
# TODO generic function types
# Only consider the common prefix of argument types.
for argt, args in zip(t.arg_types, s.arg_types):
if not is_more_precise(argt, args):
return False
return is_more_precise(t.ret_type, s.ret_type)


def infer_operator_assignment_method(typ: Type, operator: str) -> Tuple[bool, str]:
"""Determine if operator assignment on given value type is in-place, and the method name.
Expand Down Expand Up @@ -4045,3 +4026,11 @@ def is_static(func: Union[FuncBase, Decorator]) -> bool:
elif isinstance(func, FuncBase):
return func.is_static
assert False, "Unexpected func type: {}".format(type(func))


def is_subtype_no_promote(left: Type, right: Type) -> bool:
return is_subtype(left, right, ignore_promotions=True)


def is_overlapping_types_no_promote(left: Type, right: Type) -> bool:
return is_overlapping_types(left, right, ignore_promotions=True)
Loading

0 comments on commit dd70710

Please sign in to comment.