Sync with importlib_metadata 6.5 (GH-103584)

Doc/library/importlib.metadata.rst

@@ -308,6 +308,10 @@ Python module or `Import Package <https://packaging.python.org/en/latest/glossar
     >>> packages_distributions()
     {'importlib_metadata': ['importlib-metadata'], 'yaml': ['PyYAML'], 'jaraco': ['jaraco.classes', 'jaraco.functools'], ...}
 
+Some editable installs, `do not supply top-level names
+<https://github.com/pypa/packaging-problems/issues/609>`_, and thus this
+function is not reliable with such installs.
+
 .. versionadded:: 3.10
 
 .. _distributions:

Lib/importlib/metadata/__init__.py

@@ -12,7 +12,9 @@
 import functools
 import itertools
 import posixpath
+import contextlib
 import collections
+import inspect
 
 from . import _adapters, _meta
 from ._collections import FreezableDefaultDict, Pair
@@ -24,7 +26,7 @@
 from importlib import import_module
 from importlib.abc import MetaPathFinder
 from itertools import starmap
-from typing import List, Mapping, Optional
+from typing import List, Mapping, Optional, cast
 
 
 __all__ = [
@@ -341,11 +343,30 @@ def __repr__(self):
         return f'<FileHash mode: {self.mode} value: {self.value}>'
 
 
-class Distribution:
+class DeprecatedNonAbstract:
+    def __new__(cls, *args, **kwargs):
+        all_names = {
+            name for subclass in inspect.getmro(cls) for name in vars(subclass)
+        }
+        abstract = {
+            name
+            for name in all_names
+            if getattr(getattr(cls, name), '__isabstractmethod__', False)
+        }
+        if abstract:
+            warnings.warn(
+                f"Unimplemented abstract methods {abstract}",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+        return super().__new__(cls)
+
+
+class Distribution(DeprecatedNonAbstract):
     """A Python distribution package."""
 
     @abc.abstractmethod
-    def read_text(self, filename):
+    def read_text(self, filename) -> Optional[str]:
         """Attempt to load metadata file given by the name.
 
         :param filename: The name of the file in the distribution info.
@@ -419,14 +440,15 @@ def metadata(self) -> _meta.PackageMetadata:
         The returned object will have keys that name the various bits of
         metadata.  See PEP 566 for details.
         """
-        text = (
+        opt_text = (
             self.read_text('METADATA')
             or self.read_text('PKG-INFO')
             # This last clause is here to support old egg-info files.  Its
             # effect is to just end up using the PathDistribution's self._path
             # (which points to the egg-info file) attribute unchanged.
             or self.read_text('')
         )
+        text = cast(str, opt_text)
         return _adapters.Message(email.message_from_string(text))
 
     @property
@@ -455,8 +477,8 @@ def files(self):
         :return: List of PackagePath for this distribution or None
 
         Result is `None` if the metadata file that enumerates files
-        (i.e. RECORD for dist-info or SOURCES.txt for egg-info) is
-        missing.
+        (i.e. RECORD for dist-info, or installed-files.txt or
+        SOURCES.txt for egg-info) is missing.
         Result may be empty if the metadata exists but is empty.
         """
 
@@ -469,9 +491,19 @@ def make_file(name, hash=None, size_str=None):
 
         @pass_none
         def make_files(lines):
-            return list(starmap(make_file, csv.reader(lines)))
+            return starmap(make_file, csv.reader(lines))
 
-        return make_files(self._read_files_distinfo() or self._read_files_egginfo())
+        @pass_none
+        def skip_missing_files(package_paths):
+            return list(filter(lambda path: path.locate().exists(), package_paths))
+
+        return skip_missing_files(
+            make_files(
+                self._read_files_distinfo()
+                or self._read_files_egginfo_installed()
+                or self._read_files_egginfo_sources()
+            )
+        )
 
     def _read_files_distinfo(self):
         """
@@ -480,10 +512,43 @@ def _read_files_distinfo(self):
         text = self.read_text('RECORD')
         return text and text.splitlines()
 
-    def _read_files_egginfo(self):
+    def _read_files_egginfo_installed(self):
+        """
+        Read installed-files.txt and return lines in a similar
+        CSV-parsable format as RECORD: each file must be placed
+        relative to the site-packages directory, and must also be
+        quoted (since file names can contain literal commas).
+
+        This file is written when the package is installed by pip,
+        but it might not be written for other installation methods.
+        Hence, even if we can assume that this file is accurate
+        when it exists, we cannot assume that it always exists.
         """
-        SOURCES.txt might contain literal commas, so wrap each line
-        in quotes.
+        text = self.read_text('installed-files.txt')
+        # We need to prepend the .egg-info/ subdir to the lines in this file.
+        # But this subdir is only available in the PathDistribution's self._path
+        # which is not easily accessible from this base class...
+        subdir = getattr(self, '_path', None)
+        if not text or not subdir:
+            return
+        with contextlib.suppress(Exception):
+            ret = [
+                str((subdir / line).resolve().relative_to(self.locate_file('')))
+                for line in text.splitlines()
+            ]
+            return map('"{}"'.format, ret)
+
+    def _read_files_egginfo_sources(self):
+        """
+        Read SOURCES.txt and return lines in a similar CSV-parsable
+        format as RECORD: each file name must be quoted (since it
+        might contain literal commas).
+
+        Note that SOURCES.txt is not a reliable source for what
+        files are installed by a package. This file is generated
+        for a source archive, and the files that are present
+        there (e.g. setup.py) may not correctly reflect the files
+        that are present after the package has been installed.
         """
         text = self.read_text('SOURCES.txt')
         return text and map('"{}"'.format, text.splitlines())
@@ -886,8 +951,13 @@ def _top_level_declared(dist):
 
 
 def _top_level_inferred(dist):
-    return {
-        f.parts[0] if len(f.parts) > 1 else f.with_suffix('').name
+    opt_names = {
+        f.parts[0] if len(f.parts) > 1 else inspect.getmodulename(f)
         for f in always_iterable(dist.files)
-        if f.suffix == ".py"
     }
+
+    @pass_none
+    def importable_name(name):
+        return '.' not in name
+
+    return filter(importable_name, opt_names)

Lib/importlib/metadata/_adapters.py

@@ -1,10 +1,21 @@
+import functools
+import warnings
 import re
 import textwrap
 import email.message
 
 from ._text import FoldedCase
 
 
+# Do not remove prior to 2024-01-01 or Python 3.14
+_warn = functools.partial(
+    warnings.warn,
+    "Implicit None on return values is deprecated and will raise KeyErrors.",
+    DeprecationWarning,
+    stacklevel=2,
+)
+
+
 class Message(email.message.Message):
     multiple_use_keys = set(
         map(
@@ -39,6 +50,16 @@ def __init__(self, *args, **kwargs):
     def __iter__(self):
         return super().__iter__()
 
+    def __getitem__(self, item):
+        """
+        Warn users that a ``KeyError`` can be expected when a
+        mising key is supplied. Ref python/importlib_metadata#371.
+        """
+        res = super().__getitem__(item)
+        if res is None:
+            _warn()
+        return res
+
     def _repair_headers(self):
         def redent(value):
             "Correct for RFC822 indentation"

Lib/importlib/metadata/_meta.py

@@ -1,4 +1,5 @@
-from typing import Any, Dict, Iterator, List, Protocol, TypeVar, Union
+from typing import Protocol
+from typing import Any, Dict, Iterator, List, Optional, TypeVar, Union, overload
 
 
 _T = TypeVar("_T")
@@ -17,7 +18,21 @@ def __getitem__(self, key: str) -> str:
     def __iter__(self) -> Iterator[str]:
         ...  # pragma: no cover
 
-    def get_all(self, name: str, failobj: _T = ...) -> Union[List[Any], _T]:
+    @overload
+    def get(self, name: str, failobj: None = None) -> Optional[str]:
+        ...  # pragma: no cover
+
+    @overload
+    def get(self, name: str, failobj: _T) -> Union[str, _T]:
+        ...  # pragma: no cover
+
+    # overload per python/importlib_metadata#435
+    @overload
+    def get_all(self, name: str, failobj: None = None) -> Optional[List[Any]]:
+        ...  # pragma: no cover
+
+    @overload
+    def get_all(self, name: str, failobj: _T) -> Union[List[Any], _T]:
         """
         Return all values associated with a possibly multi-valued key.
         """
@@ -29,18 +44,19 @@ def json(self) -> Dict[str, Union[str, List[str]]]:
         """
 
 
-class SimplePath(Protocol):
+class SimplePath(Protocol[_T]):
     """
     A minimal subset of pathlib.Path required by PathDistribution.
     """
 
-    def joinpath(self) -> 'SimplePath':
+    def joinpath(self) -> _T:
         ...  # pragma: no cover
 
-    def __truediv__(self) -> 'SimplePath':
+    def __truediv__(self, other: Union[str, _T]) -> _T:
         ...  # pragma: no cover
 
-    def parent(self) -> 'SimplePath':
+    @property
+    def parent(self) -> _T:
         ...  # pragma: no cover
 
     def read_text(self) -> str:

Lib/test/test_importlib/_context.py

@@ -0,0 +1,13 @@
+import contextlib
+
+
+# from jaraco.context 4.3
+class suppress(contextlib.suppress, contextlib.ContextDecorator):
+    """
+    A version of contextlib.suppress with decorator support.
+
+    >>> @suppress(KeyError)
+    ... def key_error():
+    ...     {}['']
+    >>> key_error()
+    """

Lib/test/test_importlib/_path.py

@@ -0,0 +1,109 @@
+# from jaraco.path 3.5
+
+import functools
+import pathlib
+from typing import Dict, Union
+
+try:
+    from typing import Protocol, runtime_checkable
+except ImportError:  # pragma: no cover
+    # Python 3.7
+    from typing_extensions import Protocol, runtime_checkable  # type: ignore
+
+
+FilesSpec = Dict[str, Union[str, bytes, 'FilesSpec']]  # type: ignore
+
+
+@runtime_checkable
+class TreeMaker(Protocol):
+    def __truediv__(self, *args, **kwargs):
+        ...  # pragma: no cover
+
+    def mkdir(self, **kwargs):
+        ...  # pragma: no cover
+
+    def write_text(self, content, **kwargs):
+        ...  # pragma: no cover
+
+    def write_bytes(self, content):
+        ...  # pragma: no cover
+
+
+def _ensure_tree_maker(obj: Union[str, TreeMaker]) -> TreeMaker:
+    return obj if isinstance(obj, TreeMaker) else pathlib.Path(obj)  # type: ignore
+
+
+def build(
+    spec: FilesSpec,
+    prefix: Union[str, TreeMaker] = pathlib.Path(),  # type: ignore
+):
+    """
+    Build a set of files/directories, as described by the spec.
+
+    Each key represents a pathname, and the value represents
+    the content. Content may be a nested directory.
+
+    >>> spec = {
+    ...     'README.txt': "A README file",
+    ...     "foo": {
+    ...         "__init__.py": "",
+    ...         "bar": {
+    ...             "__init__.py": "",
+    ...         },
+    ...         "baz.py": "# Some code",
+    ...     }
+    ... }
+    >>> target = getfixture('tmp_path')
+    >>> build(spec, target)
+    >>> target.joinpath('foo/baz.py').read_text(encoding='utf-8')
+    '# Some code'
+    """
+    for name, contents in spec.items():
+        create(contents, _ensure_tree_maker(prefix) / name)
+
+
+@functools.singledispatch
+def create(content: Union[str, bytes, FilesSpec], path):
+    path.mkdir(exist_ok=True)
+    build(content, prefix=path)  # type: ignore
+
+
+@create.register
+def _(content: bytes, path):
+    path.write_bytes(content)
+
+
+@create.register
+def _(content: str, path):
+    path.write_text(content, encoding='utf-8')
+
+
+@create.register
+def _(content: str, path):
+    path.write_text(content, encoding='utf-8')
+
+
+class Recording:
+    """
+    A TreeMaker object that records everything that would be written.
+
+    >>> r = Recording()
+    >>> build({'foo': {'foo1.txt': 'yes'}, 'bar.txt': 'abc'}, r)
+    >>> r.record
+    ['foo/foo1.txt', 'bar.txt']
+    """
+
+    def __init__(self, loc=pathlib.PurePosixPath(), record=None):
+        self.loc = loc
+        self.record = record if record is not None else []
+
+    def __truediv__(self, other):
+        return Recording(self.loc / other, self.record)
+
+    def write_text(self, content, **kwargs):
+        self.record.append(str(self.loc))
+
+    write_bytes = write_text
+
+    def mkdir(self, **kwargs):
+        return