diff --git a/mypy/build.py b/mypy/build.py index ac6471d2383f..ff9b48d2d7b4 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -2143,7 +2143,8 @@ def parse_file(self, *, temporary: bool = False) -> None: raise CompileError( [ "mypy: can't read file '{}': {}".format( - self.path, os.strerror(ioerr.errno) + self.path.replace(os.getcwd() + os.sep, ""), + os.strerror(ioerr.errno), ) ], module_with_blocker=self.id, @@ -2861,7 +2862,7 @@ def log_configuration(manager: BuildManager, sources: list[BuildSource]) -> None manager.log(f"{'Found source:':24}{source}") # Complete list of searched paths can get very long, put them under TRACE - for path_type, paths in manager.search_paths._asdict().items(): + for path_type, paths in manager.search_paths.asdict().items(): if not paths: manager.trace(f"No {path_type}") continue diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index 774ce4f8f9b8..9b15f2aff90e 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -13,7 +13,7 @@ import subprocess import sys from enum import Enum, unique -from typing import Dict, Final, List, NamedTuple, Optional, Tuple, Union +from typing import Dict, Final, List, Optional, Tuple, Union from typing_extensions import TypeAlias as _TypeAlias from mypy import pyinfo @@ -21,16 +21,35 @@ from mypy.fscache import FileSystemCache from mypy.nodes import MypyFile from mypy.options import Options -from mypy.stubinfo import approved_stub_package_exists +from mypy.stubinfo import stub_distribution_name from mypy.util import os_path_join # Paths to be searched in find_module(). -class SearchPaths(NamedTuple): - python_path: tuple[str, ...] # where user code is found - mypy_path: tuple[str, ...] # from $MYPYPATH or config variable - package_path: tuple[str, ...] # from get_site_packages_dirs() - typeshed_path: tuple[str, ...] # paths in typeshed +class SearchPaths: + def __init__( + self, + python_path: tuple[str, ...], + mypy_path: tuple[str, ...], + package_path: tuple[str, ...], + typeshed_path: tuple[str, ...], + ) -> None: + # where user code is found + self.python_path = tuple(map(os.path.abspath, python_path)) + # from $MYPYPATH or config variable + self.mypy_path = tuple(map(os.path.abspath, mypy_path)) + # from get_site_packages_dirs() + self.package_path = tuple(map(os.path.abspath, package_path)) + # paths in typeshed + self.typeshed_path = tuple(map(os.path.abspath, typeshed_path)) + + def asdict(self) -> dict[str, tuple[str, ...]]: + return { + "python_path": self.python_path, + "mypy_path": self.mypy_path, + "package_path": self.package_path, + "typeshed_path": self.typeshed_path, + } # Package dirs are a two-tuple of path to search and whether to verify the module @@ -239,17 +258,17 @@ def find_module_via_source_set(self, id: str) -> ModuleSearchResult | None: return None def find_lib_path_dirs(self, id: str, lib_path: tuple[str, ...]) -> PackageDirs: - """Find which elements of a lib_path have the directory a module needs to exist. - - This is run for the python_path, mypy_path, and typeshed_path search paths. - """ + """Find which elements of a lib_path have the directory a module needs to exist.""" components = id.split(".") dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' dirs = [] for pathitem in self.get_toplevel_possibilities(lib_path, components[0]): # e.g., '/usr/lib/python3.4/foo/bar' - dir = os.path.normpath(os_path_join(pathitem, dir_chain)) + if dir_chain: + dir = os_path_join(pathitem, dir_chain) + else: + dir = pathitem if self.fscache.isdir(dir): dirs.append((dir, True)) return dirs @@ -418,7 +437,6 @@ def _find_module(self, id: str, use_typeshed: bool) -> ModuleSearchResult: for package_dir in self.find_lib_path_dirs(component, self.search_paths.package_path) } for pkg_dir in self.search_paths.package_path: - pkg_dir = os.path.normpath(pkg_dir) if pkg_dir not in candidate_package_dirs: continue stub_name = components[0] + "-stubs" @@ -551,8 +569,22 @@ def _find_module(self, id: str, use_typeshed: bool) -> ModuleSearchResult: if ancestor is not None: return ancestor - if approved_stub_package_exists(id): - return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED + approved_dist_name = stub_distribution_name(id) + if approved_dist_name: + if len(components) == 1: + return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED + # If we're a missing submodule of an already installed approved stubs, we don't want to + # error with APPROVED_STUBS_NOT_INSTALLED, but rather want to return NOT_FOUND. + for i in range(1, len(components)): + parent_id = ".".join(components[:i]) + if stub_distribution_name(parent_id) == approved_dist_name: + break + else: + return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED + if self.find_module(parent_id) is ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED: + return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED + return ModuleNotFoundReason.NOT_FOUND + if found_possible_third_party_missing_type_hints: return ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS return ModuleNotFoundReason.NOT_FOUND @@ -835,7 +867,6 @@ def compute_search_paths( return SearchPaths( python_path=tuple(reversed(python_path)), mypy_path=tuple(mypypath), - # package_path and typeshed_path must be normalised and absolute via os.path.abspath package_path=tuple(sys_path + site_packages), typeshed_path=tuple(lib_path), ) diff --git a/mypy/strconv.py b/mypy/strconv.py index d2ac71412f90..2d595d4b67b0 100644 --- a/mypy/strconv.py +++ b/mypy/strconv.py @@ -112,7 +112,7 @@ def visit_mypy_file(self, o: mypy.nodes.MypyFile) -> str: if o.path != "main": # Insert path. Normalize directory separators to / to unify test # case# output in all platforms. - a.insert(0, o.path.replace(os.sep, "/")) + a.insert(0, o.path.replace(os.getcwd() + os.sep, "").replace(os.sep, "/")) if o.ignored_lines: a.append("IgnoredLines(%s)" % ", ".join(str(line) for line in sorted(o.ignored_lines))) return self.dump(a, o) diff --git a/mypy/stubinfo.py b/mypy/stubinfo.py index 0ec64b037fee..8d89a2a4bede 100644 --- a/mypy/stubinfo.py +++ b/mypy/stubinfo.py @@ -6,22 +6,6 @@ def is_module_from_legacy_bundled_package(module: str) -> bool: return top_level in legacy_bundled_packages -def approved_stub_package_exists(module: str) -> bool: - top_level = module.split(".", 1)[0] - if top_level in legacy_bundled_packages: - return True - if top_level in non_bundled_packages_flat: - return True - if top_level in non_bundled_packages_namespace: - namespace = non_bundled_packages_namespace[top_level] - components = module.split(".") - for i in range(len(components), 0, -1): - module = ".".join(components[:i]) - if module in namespace: - return True - return False - - def stub_distribution_name(module: str) -> str | None: top_level = module.split(".", 1)[0] diff --git a/mypy/test/testmodulefinder.py b/mypy/test/testmodulefinder.py index bec0e60c551f..65d9a66c5fa0 100644 --- a/mypy/test/testmodulefinder.py +++ b/mypy/test/testmodulefinder.py @@ -53,12 +53,12 @@ def test__no_namespace_packages__find_a_in_pkg1(self) -> None: Find find pkg1/a.py for "a" with namespace_packages False. """ found_module = self.fmc_nons.find_module("a") - expected = os.path.join(data_path, "pkg1", "a.py") + expected = os.path.abspath(os.path.join(data_path, "pkg1", "a.py")) assert_equal(expected, found_module) def test__no_namespace_packages__find_b_in_pkg2(self) -> None: found_module = self.fmc_ns.find_module("b") - expected = os.path.join(data_path, "pkg2", "b", "__init__.py") + expected = os.path.abspath(os.path.join(data_path, "pkg2", "b", "__init__.py")) assert_equal(expected, found_module) def test__find_nsx_as_namespace_pkg_in_pkg1(self) -> None: @@ -67,7 +67,7 @@ def test__find_nsx_as_namespace_pkg_in_pkg1(self) -> None: the path to the first one found in mypypath. """ found_module = self.fmc_ns.find_module("nsx") - expected = os.path.join(data_path, "nsx-pkg1", "nsx") + expected = os.path.abspath(os.path.join(data_path, "nsx-pkg1", "nsx")) assert_equal(expected, found_module) def test__find_nsx_a_init_in_pkg1(self) -> None: @@ -75,7 +75,7 @@ def test__find_nsx_a_init_in_pkg1(self) -> None: Find nsx-pkg1/nsx/a/__init__.py for "nsx.a" in namespace mode. """ found_module = self.fmc_ns.find_module("nsx.a") - expected = os.path.join(data_path, "nsx-pkg1", "nsx", "a", "__init__.py") + expected = os.path.abspath(os.path.join(data_path, "nsx-pkg1", "nsx", "a", "__init__.py")) assert_equal(expected, found_module) def test__find_nsx_b_init_in_pkg2(self) -> None: @@ -83,7 +83,7 @@ def test__find_nsx_b_init_in_pkg2(self) -> None: Find nsx-pkg2/nsx/b/__init__.py for "nsx.b" in namespace mode. """ found_module = self.fmc_ns.find_module("nsx.b") - expected = os.path.join(data_path, "nsx-pkg2", "nsx", "b", "__init__.py") + expected = os.path.abspath(os.path.join(data_path, "nsx-pkg2", "nsx", "b", "__init__.py")) assert_equal(expected, found_module) def test__find_nsx_c_c_in_pkg3(self) -> None: @@ -91,7 +91,7 @@ def test__find_nsx_c_c_in_pkg3(self) -> None: Find nsx-pkg3/nsx/c/c.py for "nsx.c.c" in namespace mode. """ found_module = self.fmc_ns.find_module("nsx.c.c") - expected = os.path.join(data_path, "nsx-pkg3", "nsx", "c", "c.py") + expected = os.path.abspath(os.path.join(data_path, "nsx-pkg3", "nsx", "c", "c.py")) assert_equal(expected, found_module) def test__find_nsy_a__init_pyi(self) -> None: @@ -99,7 +99,7 @@ def test__find_nsy_a__init_pyi(self) -> None: Prefer nsy-pkg1/a/__init__.pyi file over __init__.py. """ found_module = self.fmc_ns.find_module("nsy.a") - expected = os.path.join(data_path, "nsy-pkg1", "nsy", "a", "__init__.pyi") + expected = os.path.abspath(os.path.join(data_path, "nsy-pkg1", "nsy", "a", "__init__.pyi")) assert_equal(expected, found_module) def test__find_nsy_b__init_py(self) -> None: @@ -109,7 +109,7 @@ def test__find_nsy_b__init_py(self) -> None: a package is preferred over a module. """ found_module = self.fmc_ns.find_module("nsy.b") - expected = os.path.join(data_path, "nsy-pkg2", "nsy", "b", "__init__.py") + expected = os.path.abspath(os.path.join(data_path, "nsy-pkg2", "nsy", "b", "__init__.py")) assert_equal(expected, found_module) def test__find_nsy_c_pyi(self) -> None: @@ -119,17 +119,17 @@ def test__find_nsy_c_pyi(self) -> None: .pyi is preferred over .py. """ found_module = self.fmc_ns.find_module("nsy.c") - expected = os.path.join(data_path, "nsy-pkg2", "nsy", "c.pyi") + expected = os.path.abspath(os.path.join(data_path, "nsy-pkg2", "nsy", "c.pyi")) assert_equal(expected, found_module) def test__find_a_in_pkg1(self) -> None: found_module = self.fmc_ns.find_module("a") - expected = os.path.join(data_path, "pkg1", "a.py") + expected = os.path.abspath(os.path.join(data_path, "pkg1", "a.py")) assert_equal(expected, found_module) def test__find_b_init_in_pkg2(self) -> None: found_module = self.fmc_ns.find_module("b") - expected = os.path.join(data_path, "pkg2", "b", "__init__.py") + expected = os.path.abspath(os.path.join(data_path, "pkg2", "b", "__init__.py")) assert_equal(expected, found_module) def test__find_d_nowhere(self) -> None: @@ -165,7 +165,7 @@ def setUp(self) -> None: self.fmc_nons = FindModuleCache(self.search_paths, fscache=None, options=options) def path(self, *parts: str) -> str: - return os.path.normpath(os.path.join(self.package_dir, *parts)) + return os.path.abspath(os.path.join(self.package_dir, *parts)) def test__packages_with_ns(self) -> None: cases = [ @@ -214,7 +214,7 @@ def test__packages_with_ns(self) -> None: # A regular package with an installed set of stubs ("foo.bar", self.path("foo-stubs", "bar.pyi")), # A regular, non-site-packages module - ("a", os.path.join(data_path, "pkg1", "a.py")), + ("a", os.path.abspath(os.path.join(data_path, "pkg1", "a.py"))), ] for module, expected in cases: template = "Find(" + module + ") got {}; expected {}" @@ -269,7 +269,7 @@ def test__packages_without_ns(self) -> None: # A regular package with an installed set of stubs ("foo.bar", self.path("foo-stubs", "bar.pyi")), # A regular, non-site-packages module - ("a", os.path.join(data_path, "pkg1", "a.py")), + ("a", os.path.abspath(os.path.join(data_path, "pkg1", "a.py"))), ] for module, expected in cases: template = "Find(" + module + ") got {}; expected {}" diff --git a/mypy/test/teststubinfo.py b/mypy/test/teststubinfo.py index 10ce408e7023..518194d35e1d 100644 --- a/mypy/test/teststubinfo.py +++ b/mypy/test/teststubinfo.py @@ -3,7 +3,6 @@ import unittest from mypy.stubinfo import ( - approved_stub_package_exists, is_module_from_legacy_bundled_package, legacy_bundled_packages, non_bundled_packages_flat, @@ -18,17 +17,6 @@ def test_is_legacy_bundled_packages(self) -> None: assert is_module_from_legacy_bundled_package("pycurl") assert is_module_from_legacy_bundled_package("dataclasses") - def test_approved_stub_package_exists(self) -> None: - assert not approved_stub_package_exists("foobar_asdf") - assert approved_stub_package_exists("pycurl") - assert approved_stub_package_exists("babel") - assert approved_stub_package_exists("google.cloud.ndb") - assert approved_stub_package_exists("google.cloud.ndb.submodule") - assert not approved_stub_package_exists("google.cloud.unknown") - assert approved_stub_package_exists("google.protobuf") - assert approved_stub_package_exists("google.protobuf.submodule") - assert not approved_stub_package_exists("google") - def test_stub_distribution_name(self) -> None: assert stub_distribution_name("foobar_asdf") is None assert stub_distribution_name("pycurl") == "types-pycurl" diff --git a/test-data/unit/check-modules.test b/test-data/unit/check-modules.test index f368f244eb34..68897790e4bf 100644 --- a/test-data/unit/check-modules.test +++ b/test-data/unit/check-modules.test @@ -3136,23 +3136,15 @@ import google.cloud.ndb # E: Library stubs not installed for "google.cloud.ndb" from google.cloud import ndb [case testMissingSubmoduleOfInstalledStubPackage] -import bleach.xyz -from bleach.abc import fgh +import bleach.exists +import bleach.xyz # E: Cannot find implementation or library stub for module named "bleach.xyz" \ + # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +from bleach.abc import fgh # E: Cannot find implementation or library stub for module named "bleach.abc" [file bleach/__init__.pyi] -[out] -main:1: error: Library stubs not installed for "bleach.xyz" -main:1: note: Hint: "python3 -m pip install types-bleach" -main:1: note: (or run "mypy --install-types" to install all missing stub packages) -main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports -main:2: error: Library stubs not installed for "bleach.abc" +[file bleach/exists.pyi] -[case testMissingSubmoduleOfInstalledStubPackageIgnored-xfail] +[case testMissingSubmoduleOfInstalledStubPackageIgnored] # flags: --ignore-missing-imports - -# TODO: testMissingSubmoduleOfInstalledStubPackageIgnored was regressed in -# https://github.com/python/mypy/pull/15347 but didn't cause failures because we don't have a -# package path in this unit test - import bleach.xyz from bleach.abc import fgh [file bleach/__init__.pyi]