diff --git a/mypy/checker.py b/mypy/checker.py index 64b3313d40bda..b01b4bbf984ca 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -5,7 +5,8 @@ from contextlib import contextmanager from typing import ( - Dict, Set, List, cast, Tuple, TypeVar, Union, Optional, NamedTuple, Iterator, Iterable, Any + Dict, Set, List, cast, Tuple, TypeVar, Union, Optional, NamedTuple, Iterator, Iterable, + Sequence ) from mypy.errors import Errors, report_internal_error @@ -72,19 +73,31 @@ DEFAULT_LAST_PASS = 1 # type: Final # Pass numbers start at 0 +DeferredNodeType = Union[FuncDef, LambdaExpr, OverloadedFuncDef, Decorator] +FineGrainedDeferredNodeType = Union[FuncDef, MypyFile, OverloadedFuncDef] # A node which is postponed to be processed during the next pass. -# This is used for both batch mode and fine-grained incremental mode. +# In normal mode one can defer functions and methods (also decorated and/or overloaded) +# and lambda expressions. Nested functions can't be deferred -- only top-level functions +# and methods of classes not defined within a function can be deferred. DeferredNode = NamedTuple( 'DeferredNode', [ - # In batch mode only FuncDef and LambdaExpr are supported - ('node', Union[FuncDef, LambdaExpr, MypyFile, OverloadedFuncDef]), + ('node', DeferredNodeType), ('context_type_name', Optional[str]), # Name of the surrounding class (for error messages) ('active_typeinfo', Optional[TypeInfo]), # And its TypeInfo (for semantic analysis # self type handling) ]) +# Same as above, but for fine-grained mode targets. Only top-level functions/methods +# and module top levels are allowed as such. +FineGrainedDeferredNode = NamedTuple( + 'FineDeferredNode', + [ + ('node', FineGrainedDeferredNodeType), + ('context_type_name', Optional[str]), + ('active_typeinfo', Optional[TypeInfo]), + ]) # Data structure returned by find_isinstance_check representing # information learned from the truth or falsehood of a condition. The @@ -283,7 +296,10 @@ def check_first_pass(self) -> None: self.tscope.leave() - def check_second_pass(self, todo: Optional[List[DeferredNode]] = None) -> bool: + def check_second_pass(self, + todo: Optional[Sequence[Union[DeferredNode, + FineGrainedDeferredNode]]] = None + ) -> bool: """Run second or following pass of type checking. This goes through deferred nodes, returning True if there were any. @@ -300,7 +316,7 @@ def check_second_pass(self, todo: Optional[List[DeferredNode]] = None) -> bool: else: assert not self.deferred_nodes self.deferred_nodes = [] - done = set() # type: Set[Union[FuncDef, LambdaExpr, MypyFile, OverloadedFuncDef]] + done = set() # type: Set[Union[DeferredNodeType, FineGrainedDeferredNodeType]] for node, type_name, active_typeinfo in todo: if node in done: continue @@ -314,10 +330,7 @@ def check_second_pass(self, todo: Optional[List[DeferredNode]] = None) -> bool: self.tscope.leave() return True - def check_partial(self, node: Union[FuncDef, - LambdaExpr, - MypyFile, - OverloadedFuncDef]) -> None: + def check_partial(self, node: Union[DeferredNodeType, FineGrainedDeferredNodeType]) -> None: if isinstance(node, MypyFile): self.check_top_level(node) else: @@ -338,6 +351,23 @@ def check_top_level(self, node: MypyFile) -> None: assert not self.current_node_deferred # TODO: Handle __all__ + def defer_node(self, node: DeferredNodeType, enclosing_class: Optional[TypeInfo]) -> None: + """Defer a node for processing during next type-checking pass. + + Args: + node: function/method being deferred + enclosing_class: for methods, the class where the method is defined + NOTE: this can't handle nested functions/methods. + """ + if self.errors.type_name: + type_name = self.errors.type_name[-1] + else: + type_name = None + # We don't freeze the entire scope since only top-level functions and methods + # can be deferred. Only module/class level scope information is needed. + # Module-level scope information is preserved in the TypeChecker instance. + self.deferred_nodes.append(DeferredNode(node, type_name, enclosing_class)) + def handle_cannot_determine_type(self, name: str, context: Context) -> None: node = self.scope.top_non_lambda_function() if self.pass_num < self.last_pass and isinstance(node, FuncDef): @@ -345,13 +375,8 @@ def handle_cannot_determine_type(self, name: str, context: Context) -> None: # lambdas because they are coupled to the surrounding function # through the binder and the inferred type of the lambda, so it # would get messy. - if self.errors.type_name: - type_name = self.errors.type_name[-1] - else: - type_name = None - # Shouldn't we freeze the entire scope? enclosing_class = self.scope.enclosing_class() - self.deferred_nodes.append(DeferredNode(node, type_name, enclosing_class)) + self.defer_node(node, enclosing_class) # Set a marker so that we won't infer additional types in this # function. Any inferred types could be bogus, because there's at # least one type that we don't know. @@ -1253,15 +1278,26 @@ def expand_typevars(self, defn: FuncItem, else: return [(defn, typ)] - def check_method_override(self, defn: Union[FuncBase, Decorator]) -> None: - """Check if function definition is compatible with base classes.""" + def check_method_override(self, defn: Union[FuncDef, OverloadedFuncDef, Decorator]) -> None: + """Check if function definition is compatible with base classes. + + This may defer the method if a signature is not available in at least one base class. + """ # Check against definitions in base classes. for base in defn.info.mro[1:]: - self.check_method_or_accessor_override_for_base(defn, base) + if self.check_method_or_accessor_override_for_base(defn, base): + # Node was deferred, we will have another attempt later. + return + + def check_method_or_accessor_override_for_base(self, defn: Union[FuncDef, + OverloadedFuncDef, + Decorator], + base: TypeInfo) -> bool: + """Check if method definition is compatible with a base class. - def check_method_or_accessor_override_for_base(self, defn: Union[FuncBase, Decorator], - base: TypeInfo) -> None: - """Check if method definition is compatible with a base class.""" + Return True if the node was deferred because one of the corresponding + superclass nodes is not ready. + """ if base: name = defn.name() base_attr = base.names.get(name) @@ -1277,7 +1313,8 @@ def check_method_or_accessor_override_for_base(self, defn: Union[FuncBase, Decor if name not in ('__init__', '__new__', '__init_subclass__'): # Check method override # (__init__, __new__, __init_subclass__ are special). - self.check_method_override_for_base_with_name(defn, name, base) + if self.check_method_override_for_base_with_name(defn, name, base): + return True if name in nodes.inplace_operator_methods: # Figure out the name of the corresponding operator method. method = '__' + name[3:] @@ -1285,11 +1322,17 @@ def check_method_or_accessor_override_for_base(self, defn: Union[FuncBase, Decor # always introduced safely if a base class defined __add__. # TODO can't come up with an example where this is # necessary; now it's "just in case" - self.check_method_override_for_base_with_name(defn, method, - base) + return self.check_method_override_for_base_with_name(defn, method, + base) + return False def check_method_override_for_base_with_name( - self, defn: Union[FuncBase, Decorator], name: str, base: TypeInfo) -> None: + self, defn: Union[FuncDef, OverloadedFuncDef, Decorator], + name: str, base: TypeInfo) -> bool: + """Check if overriding an attribute `name` of `base` with `defn` is valid. + + Return True if the supertype node was not analysed yet, and `defn` was deferred. + """ base_attr = base.names.get(name) if base_attr: # The name of the method is defined in the base class. @@ -1302,7 +1345,7 @@ def check_method_override_for_base_with_name( context = defn.func # Construct the type of the overriding method. - if isinstance(defn, FuncBase): + if isinstance(defn, (FuncDef, OverloadedFuncDef)): typ = self.function_type(defn) # type: Type override_class_or_static = defn.is_class or defn.is_static else: @@ -1317,13 +1360,18 @@ def check_method_override_for_base_with_name( original_type = base_attr.type original_node = base_attr.node if original_type is None: - if isinstance(original_node, FuncBase): + if self.pass_num < self.last_pass: + # If there are passes left, defer this node until next pass, + # otherwise try reconstructing the method type from available information. + self.defer_node(defn, defn.info) + return True + elif isinstance(original_node, (FuncDef, OverloadedFuncDef)): original_type = self.function_type(original_node) elif isinstance(original_node, Decorator): original_type = self.function_type(original_node.func) else: assert False, str(base_attr.node) - if isinstance(original_node, FuncBase): + if isinstance(original_node, (FuncDef, OverloadedFuncDef)): original_class_or_static = original_node.is_class or original_node.is_static elif isinstance(original_node, Decorator): fdef = original_node.func @@ -1359,6 +1407,7 @@ def check_method_override_for_base_with_name( else: self.msg.signature_incompatible_with_supertype( defn.name(), name, base.name(), context) + return False def get_op_other_domain(self, tp: FunctionLike) -> Optional[Type]: if isinstance(tp, CallableType): diff --git a/mypy/semanal.py b/mypy/semanal.py index 1e24dd3ea1c16..b9f9328d9496d 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -317,7 +317,7 @@ def visit_file(self, file_node: MypyFile, fnam: str, options: Options, del self.cur_mod_node del self.globals - def refresh_partial(self, node: Union[MypyFile, FuncItem, OverloadedFuncDef], + def refresh_partial(self, node: Union[MypyFile, FuncDef, OverloadedFuncDef], patches: List[Tuple[int, Callable[[], None]]]) -> None: """Refresh a stale target in fine-grained incremental mode.""" self.patches = patches diff --git a/mypy/semanal_pass3.py b/mypy/semanal_pass3.py index c00da4d45f996..9b8fab138c177 100644 --- a/mypy/semanal_pass3.py +++ b/mypy/semanal_pass3.py @@ -73,7 +73,7 @@ def visit_file(self, file_node: MypyFile, fnam: str, options: Options, del self.cur_mod_node self.patches = [] - def refresh_partial(self, node: Union[MypyFile, FuncItem, OverloadedFuncDef], + def refresh_partial(self, node: Union[MypyFile, FuncDef, OverloadedFuncDef], patches: List[Tuple[int, Callable[[], None]]]) -> None: """Refresh a stale target in fine-grained incremental mode.""" self.options = self.sem.options diff --git a/mypy/server/aststrip.py b/mypy/server/aststrip.py index 5314ca7f3f5be..e109ad4ab4f31 100644 --- a/mypy/server/aststrip.py +++ b/mypy/server/aststrip.py @@ -52,7 +52,7 @@ from mypy.typestate import TypeState -def strip_target(node: Union[MypyFile, FuncItem, OverloadedFuncDef]) -> None: +def strip_target(node: Union[MypyFile, FuncDef, OverloadedFuncDef]) -> None: """Reset a fine-grained incremental target to state after semantic analysis pass 1. NOTE: Currently we opportunistically only reset changes that are known to otherwise diff --git a/mypy/server/update.py b/mypy/server/update.py index 9c306ca89f253..7943ffaee95bd 100644 --- a/mypy/server/update.py +++ b/mypy/server/update.py @@ -122,7 +122,7 @@ BuildManager, State, BuildSource, BuildResult, Graph, load_graph, process_fresh_modules, DEBUG_FINE_GRAINED, ) -from mypy.checker import DeferredNode +from mypy.checker import FineGrainedDeferredNode from mypy.errors import CompileError from mypy.nodes import ( MypyFile, FuncDef, TypeInfo, SymbolNode, Decorator, @@ -780,7 +780,7 @@ def find_targets_recursive( graph: Graph, triggers: Set[str], deps: Dict[str, Set[str]], - up_to_date_modules: Set[str]) -> Tuple[Dict[str, Set[DeferredNode]], + up_to_date_modules: Set[str]) -> Tuple[Dict[str, Set[FineGrainedDeferredNode]], Set[str], Set[TypeInfo]]: """Find names of all targets that need to reprocessed, given some triggers. @@ -788,7 +788,7 @@ def find_targets_recursive( * Dictionary from module id to a set of stale targets. * A set of module ids for unparsed modules with stale targets. """ - result = {} # type: Dict[str, Set[DeferredNode]] + result = {} # type: Dict[str, Set[FineGrainedDeferredNode]] worklist = triggers processed = set() # type: Set[str] stale_protos = set() # type: Set[TypeInfo] @@ -834,7 +834,7 @@ def find_targets_recursive( def reprocess_nodes(manager: BuildManager, graph: Dict[str, State], module_id: str, - nodeset: Set[DeferredNode], + nodeset: Set[FineGrainedDeferredNode], deps: Dict[str, Set[str]]) -> Set[str]: """Reprocess a set of nodes within a single module. @@ -850,7 +850,7 @@ def reprocess_nodes(manager: BuildManager, old_symbols = {name: names.copy() for name, names in old_symbols.items()} old_symbols_snapshot = snapshot_symbol_table(file_node.fullname(), file_node.names) - def key(node: DeferredNode) -> int: + def key(node: FineGrainedDeferredNode) -> int: # Unlike modules which are sorted by name within SCC, # nodes within the same module are sorted by line number, because # this is how they are processed in normal mode. @@ -959,7 +959,7 @@ def find_symbol_tables_recursive(prefix: str, symbols: SymbolTable) -> Dict[str, def update_deps(module_id: str, - nodes: List[DeferredNode], + nodes: List[FineGrainedDeferredNode], graph: Dict[str, State], deps: Dict[str, Set[str]], options: Options) -> None: @@ -977,7 +977,7 @@ def update_deps(module_id: str, def lookup_target(manager: BuildManager, - target: str) -> Tuple[List[DeferredNode], Optional[TypeInfo]]: + target: str) -> Tuple[List[FineGrainedDeferredNode], Optional[TypeInfo]]: """Look up a target by fully-qualified name. The first item in the return tuple is a list of deferred nodes that @@ -1025,7 +1025,7 @@ def not_found() -> None: # a deserialized TypeInfo with missing attributes. not_found() return [], None - result = [DeferredNode(file, None, None)] + result = [FineGrainedDeferredNode(file, None, None)] stale_info = None # type: Optional[TypeInfo] if node.is_protocol: stale_info = node @@ -1050,7 +1050,7 @@ def not_found() -> None: # context will be wrong and it could be a partially initialized deserialized node. not_found() return [], None - return [DeferredNode(node, active_class_name, active_class)], None + return [FineGrainedDeferredNode(node, active_class_name, active_class)], None def is_verbose(manager: BuildManager) -> bool: @@ -1058,7 +1058,7 @@ def is_verbose(manager: BuildManager) -> bool: def target_from_node(module: str, - node: Union[FuncDef, MypyFile, OverloadedFuncDef, LambdaExpr] + node: Union[FuncDef, MypyFile, OverloadedFuncDef] ) -> Optional[str]: """Return the target name corresponding to a deferred node. @@ -1073,10 +1073,8 @@ def target_from_node(module: str, # Actually a reference to another module -- likely a stale dependency. return None return module - elif isinstance(node, (OverloadedFuncDef, FuncDef)): + else: # OverloadedFuncDef or FuncDef if node.info: return '%s.%s' % (node.info.fullname(), node.name()) else: return '%s.%s' % (module, node.name()) - else: - assert False, "Lambda expressions can't be deferred in fine-grained incremental mode" diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 14ed21207a00d..f1e0f7c7c26eb 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -5117,3 +5117,295 @@ class C: def x(self) -> int: pass [builtins fixtures/property.pyi] [out] + +[case testCyclicDecorator] +import b +[file a.py] +import b +import c + +class A(b.B): + @c.deco + def meth(self) -> int: ... +[file b.py] +import a +import c + +class B: + @c.deco + def meth(self) -> int: ... +[file c.py] +from typing import TypeVar, Tuple, Callable +T = TypeVar('T') +def deco(f: Callable[..., T]) -> Callable[..., Tuple[T, int]]: ... +[out] + +[case testCyclicOverload] +import b +[file a.pyi] +import b +from typing import overload + +class A(b.B): + @overload + def meth(self, x: int) -> int: ... + @overload + def meth(self, x: str) -> str: ... +[file b.pyi] +import a +from typing import overload + +class B: + @overload + def meth(self, x: int) -> int: ... + @overload + def meth(self, x: str) -> str: ... +[out] + +[case testCyclicOverloadDeferred] +import b +[file a.py] +import b +from typing import overload, Union + +class A(b.B): + @overload + def meth(self, x: int) -> int: ... + @overload + def meth(self, x: str) -> str: ... + def meth(self, x) -> Union[int, str]: + reveal_type(other.x) # E: Revealed type is 'builtins.int' + return 0 + +other: Other +class Other: + def __init__(self) -> None: + self.x = f() +def f() -> int: ... +[file b.py] +import a +from typing import overload + +class B: + @overload + def meth(self, x: int) -> int: ... + @overload + def meth(self, x: str) -> str: ... + def meth(self, x): + pass +[out] + +[case testCyclicOverrideAny] +import a +[file b.py] +import a +class Sub(a.Base): + def x(self) -> int: pass + +[file a.py] +import b +class Base: + def __init__(self): + self.x = 1 +[out] + +[case testCyclicOverrideChecked] +import a +[file b.py] +import a +class Sub(a.Base): + def x(self) -> int: pass # E: Signature of "x" incompatible with supertype "Base" + +[file a.py] +import b +class Base: + def __init__(self) -> None: + self.x = 1 +[out] + +[case testCyclicOverrideCheckedDecorator] +import a +[file b.py] +import a +import c +class Sub(a.Base): + @c.deco + def x(self) -> int: pass # E: Signature of "x" incompatible with supertype "Base" + +[file a.py] +import b +import c +class Base: + def __init__(self) -> None: + self.x = 1 +[file c.py] +from typing import TypeVar, Tuple, Callable +T = TypeVar('T') +def deco(f: Callable[..., T]) -> Callable[..., Tuple[T, int]]: ... +[out] + +[case testCyclicOverrideCheckedDecoratorDeferred] +import a +[file b.py] +import a +import c +class Sub(a.Base): + @c.deco + def x(self) -> int: pass # E: Signature of "x" incompatible with supertype "Base" + +[file a.py] +import b +import c +class Base: + def __init__(self) -> None: + self.x = f() + +def f() -> int: ... +[file c.py] +from typing import TypeVar, Tuple, Callable +T = TypeVar('T') +def deco(f: Callable[..., T]) -> Callable[..., Tuple[T, int]]: ... +[out] + +[case testCyclicOverrideAnyDecoratorDeferred] +import a +[file b.py] +import a +import c +class Sub(a.Base): + @c.deco + def x(self) -> int: pass + +[file a.py] +from b import Sub +import c +class Base: + def __init__(self) -> None: + self.x = f() + +def f() -> int: ... +[file c.py] +from typing import Any, Callable +def deco(f: Callable[..., Any]) -> Any: ... +[out] + +[case testCyclicDecoratorDoubleDeferred] +import b +[file a.py] +import b +import c + +class A(b.B): + @c.deco + def meth(self) -> int: + reveal_type(other.x) # E: Revealed type is 'builtins.int' + return 0 + +other: Other +class Other: + def __init__(self) -> None: + self.x = f() +def f() -> int: ... +[file b.py] +from a import A +import c + +class B: + @c.deco + def meth(self) -> int: + pass +[file c.py] +from typing import TypeVar, Tuple, Callable +T = TypeVar('T') +def deco(f: Callable[..., T]) -> Callable[..., Tuple[T, int]]: ... +[out] + +[case testCyclicDecoratorSuper] +import b +[file a.py] +import b +import c + +class A(b.B): + @c.deco + def meth(self) -> int: + y = super().meth() + reveal_type(y) # E: Revealed type is 'Tuple[builtins.int*, builtins.int]' + return 0 +[file b.py] +from a import A +import c + +class B: + @c.deco + def meth(self) -> int: + pass +[file c.py] +from typing import TypeVar, Tuple, Callable +T = TypeVar('T') +def deco(f: Callable[..., T]) -> Callable[..., Tuple[T, int]]: ... +[out] + +[case testCyclicDecoratorBothDeferred] +import b +[file a.py] +import b +import c + +class A(b.B): + @c.deco + def meth(self) -> int: + pass +[file b.py] +from a import A +import c + +class B: + @c.deco + def meth(self) -> int: + reveal_type(other.x) # E: Revealed type is 'builtins.int' + return 0 + +other: Other +class Other: + def __init__(self) -> None: + self.x = f() +def f() -> int: ... +[file c.py] +from typing import TypeVar, Tuple, Callable +T = TypeVar('T') +def deco(f: Callable[..., T]) -> Callable[..., Tuple[T, int]]: ... +[out] + +[case testCyclicDecoratorSuperDeferred] +import b +[file a.py] +import b +import c + +class A(b.B): + @c.deco + def meth(self) -> int: + y = super().meth() + reveal_type(y) # E: Revealed type is 'Tuple[builtins.int*, builtins.int]' + reveal_type(other.x) # E: Revealed type is 'builtins.int' + return 0 + +other: Other +class Other: + def __init__(self) -> None: + self.x = f() +def f() -> int: ... +[file b.py] +from a import A +import c + +class B: + @c.deco + def meth(self) -> int: + pass +[file c.py] +from typing import TypeVar, Tuple, Callable +T = TypeVar('T') +def deco(f: Callable[..., T]) -> Callable[..., Tuple[T, int]]: ... +[out]