Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

importlib: improve bytes handling #9070

Merged
merged 4 commits into from
Nov 3, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 22 additions & 17 deletions stdlib/importlib/abc.pyi
Original file line number Diff line number Diff line change
@@ -1,12 +1,19 @@
import sys
import types
from _typeshed import OpenBinaryMode, OpenBinaryModeReading, OpenBinaryModeUpdating, OpenBinaryModeWriting, OpenTextMode
from _typeshed import (
OpenBinaryMode,
OpenBinaryModeReading,
OpenBinaryModeUpdating,
OpenBinaryModeWriting,
OpenTextMode,
ReadableBuffer,
)
from abc import ABCMeta, abstractmethod
from collections.abc import Iterator, Mapping, Sequence
from importlib.machinery import ModuleSpec
from io import BufferedRandom, BufferedReader, BufferedWriter, FileIO, TextIOWrapper
from typing import IO, Any, BinaryIO, NoReturn, Protocol, overload, runtime_checkable
from typing_extensions import Literal, TypeAlias
from typing_extensions import Literal

if sys.version_info >= (3, 11):
__all__ = [
Expand All @@ -24,8 +31,6 @@ if sys.version_info >= (3, 11):
"TraversableResources",
]

_Path: TypeAlias = bytes | str

class Finder(metaclass=ABCMeta): ...

class Loader(metaclass=ABCMeta):
Expand All @@ -38,7 +43,7 @@ class Loader(metaclass=ABCMeta):

class ResourceLoader(Loader):
@abstractmethod
def get_data(self, path: _Path) -> bytes: ...
def get_data(self, path: str) -> bytes: ...
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.


class InspectLoader(Loader):
def is_package(self, fullname: str) -> bool: ...
Expand All @@ -47,40 +52,40 @@ class InspectLoader(Loader):
def get_source(self, fullname: str) -> str | None: ...
def exec_module(self, module: types.ModuleType) -> None: ...
@staticmethod
def source_to_code(data: bytes | str, path: str = ...) -> types.CodeType: ...
def source_to_code(data: ReadableBuffer | str, path: str = ...) -> types.CodeType: ...
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could also be AST, but I don't think that's too important.


class ExecutionLoader(InspectLoader):
@abstractmethod
def get_filename(self, fullname: str) -> _Path: ...
def get_filename(self, fullname: str) -> str: ...
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Docstring says "Abstract method which should return the value that file is to be set to." __file__ is always a str, not bytes. (Note that _Path = str | bytes.)


class SourceLoader(ResourceLoader, ExecutionLoader, metaclass=ABCMeta):
def path_mtime(self, path: _Path) -> float: ...
def set_data(self, path: _Path, data: bytes) -> None: ...
def path_mtime(self, path: str) -> float: ...
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Docstrings say the path must be str for all three methods.

def set_data(self, path: str, data: bytes) -> None: ...
def get_source(self, fullname: str) -> str | None: ...
def path_stats(self, path: _Path) -> Mapping[str, Any]: ...
def path_stats(self, path: str) -> Mapping[str, Any]: ...

# Please keep in sync with sys._MetaPathFinder
class MetaPathFinder(Finder):
def find_module(self, fullname: str, path: Sequence[_Path] | None) -> Loader | None: ...
def find_module(self, fullname: str, path: Sequence[str] | None) -> Loader | None: ...
def invalidate_caches(self) -> None: ...
# Not defined on the actual class, but expected to exist.
def find_spec(
self, fullname: str, path: Sequence[_Path] | None, target: types.ModuleType | None = ...
self, fullname: str, path: Sequence[str] | None, target: types.ModuleType | None = ...
) -> ModuleSpec | None: ...

class PathEntryFinder(Finder):
def find_module(self, fullname: str) -> Loader | None: ...
def find_loader(self, fullname: str) -> tuple[Loader | None, Sequence[_Path]]: ...
def find_loader(self, fullname: str) -> tuple[Loader | None, Sequence[str]]: ...
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The second element is ModuleSpec.submodule_search_locations which is a list[str]. https://github.com/python/cpython/blob/2cfcaf5af602b297fc90086de4d8ac980c7891e2/Lib/importlib/abc.py#L155

def invalidate_caches(self) -> None: ...
# Not defined on the actual class, but expected to exist.
def find_spec(self, fullname: str, target: types.ModuleType | None = ...) -> ModuleSpec | None: ...

class FileLoader(ResourceLoader, ExecutionLoader, metaclass=ABCMeta):
name: str
path: _Path
def __init__(self, fullname: str, path: _Path) -> None: ...
def get_data(self, path: _Path) -> bytes: ...
def get_filename(self, name: str | None = ...) -> _Path: ...
path: str
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This class has no implementation, but from _bootstrap_external.FileLoader it seems clear the path is not intended to be bytes: https://github.com/python/cpython/blob/2cfcaf5af602b297fc90086de4d8ac980c7891e2/Lib/importlib/_bootstrap_external.py#L1195

def __init__(self, fullname: str, path: str) -> None: ...
def get_data(self, path: str) -> bytes: ...
def get_filename(self, name: str | None = ...) -> str: ...
def load_module(self, name: str | None = ...) -> types.ModuleType: ...

class ResourceReader(metaclass=ABCMeta):
Expand Down
23 changes: 12 additions & 11 deletions stdlib/importlib/machinery.pyi
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import importlib.abc
import sys
import types
from _typeshed import ReadableBuffer
from collections.abc import Callable, Iterable, Sequence
from typing import Any

Expand Down Expand Up @@ -31,10 +32,10 @@ class ModuleSpec:
class BuiltinImporter(importlib.abc.MetaPathFinder, importlib.abc.InspectLoader):
# MetaPathFinder
@classmethod
def find_module(cls, fullname: str, path: Sequence[importlib.abc._Path] | None = ...) -> importlib.abc.Loader | None: ...
def find_module(cls, fullname: str, path: Sequence[str] | None = ...) -> importlib.abc.Loader | None: ...
@classmethod
def find_spec(
cls, fullname: str, path: Sequence[importlib.abc._Path] | None = ..., target: types.ModuleType | None = ...
cls, fullname: str, path: Sequence[str] | None = ..., target: types.ModuleType | None = ...
) -> ModuleSpec | None: ...
# InspectLoader
@classmethod
Expand Down Expand Up @@ -62,10 +63,10 @@ class BuiltinImporter(importlib.abc.MetaPathFinder, importlib.abc.InspectLoader)
class FrozenImporter(importlib.abc.MetaPathFinder, importlib.abc.InspectLoader):
# MetaPathFinder
@classmethod
def find_module(cls, fullname: str, path: Sequence[importlib.abc._Path] | None = ...) -> importlib.abc.Loader | None: ...
def find_module(cls, fullname: str, path: Sequence[str] | None = ...) -> importlib.abc.Loader | None: ...
@classmethod
def find_spec(
cls, fullname: str, path: Sequence[importlib.abc._Path] | None = ..., target: types.ModuleType | None = ...
cls, fullname: str, path: Sequence[str] | None = ..., target: types.ModuleType | None = ...
) -> ModuleSpec | None: ...
# InspectLoader
@classmethod
Expand All @@ -91,10 +92,10 @@ class FrozenImporter(importlib.abc.MetaPathFinder, importlib.abc.InspectLoader):

class WindowsRegistryFinder(importlib.abc.MetaPathFinder):
@classmethod
def find_module(cls, fullname: str, path: Sequence[importlib.abc._Path] | None = ...) -> importlib.abc.Loader | None: ...
def find_module(cls, fullname: str, path: Sequence[str] | None = ...) -> importlib.abc.Loader | None: ...
@classmethod
def find_spec(
cls, fullname: str, path: Sequence[importlib.abc._Path] | None = ..., target: types.ModuleType | None = ...
cls, fullname: str, path: Sequence[str] | None = ..., target: types.ModuleType | None = ...
) -> ModuleSpec | None: ...

class PathFinder:
Expand All @@ -113,10 +114,10 @@ class PathFinder:

@classmethod
def find_spec(
cls, fullname: str, path: Sequence[bytes | str] | None = ..., target: types.ModuleType | None = ...
cls, fullname: str, path: Sequence[str] | None = ..., target: types.ModuleType | None = ...
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The path gets passed to _get_spec, which ignores any non-str entries in the path: https://github.com/python/cpython/blob/2cfcaf5af602b297fc90086de4d8ac980c7891e2/Lib/importlib/_bootstrap_external.py#L1525

) -> ModuleSpec | None: ...
@classmethod
def find_module(cls, fullname: str, path: Sequence[bytes | str] | None = ...) -> importlib.abc.Loader | None: ...
def find_module(cls, fullname: str, path: Sequence[str] | None = ...) -> importlib.abc.Loader | None: ...

SOURCE_SUFFIXES: list[str]
DEBUG_BYTECODE_SUFFIXES: list[str]
Expand All @@ -135,13 +136,13 @@ class FileFinder(importlib.abc.PathEntryFinder):
) -> Callable[[str], importlib.abc.PathEntryFinder]: ...

class SourceFileLoader(importlib.abc.FileLoader, importlib.abc.SourceLoader):
def set_data(self, path: importlib.abc._Path, data: bytes, *, _mode: int = ...) -> None: ...
def set_data(self, path: str, data: ReadableBuffer, *, _mode: int = ...) -> None: ...

class SourcelessFileLoader(importlib.abc.FileLoader, importlib.abc.SourceLoader): ...

class ExtensionFileLoader(importlib.abc.ExecutionLoader):
def __init__(self, name: str, path: importlib.abc._Path) -> None: ...
def get_filename(self, name: str | None = ...) -> importlib.abc._Path: ...
def __init__(self, name: str, path: str) -> None: ...
def get_filename(self, name: str | None = ...) -> str: ...
def get_source(self, fullname: str) -> None: ...
def create_module(self, spec: ModuleSpec) -> types.ModuleType: ...
def exec_module(self, module: types.ModuleType) -> None: ...
Expand Down
6 changes: 3 additions & 3 deletions stdlib/importlib/util.pyi
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import importlib.abc
import importlib.machinery
import types
from _typeshed import StrOrBytesPath
from _typeshed import ReadableBuffer, StrOrBytesPath
from collections.abc import Callable
from typing import Any
from typing_extensions import ParamSpec
Expand All @@ -17,7 +17,7 @@ MAGIC_NUMBER: bytes

def cache_from_source(path: str, debug_override: bool | None = ..., *, optimization: Any | None = ...) -> str: ...
def source_from_cache(path: str) -> str: ...
def decode_source(source_bytes: bytes) -> str: ...
def decode_source(source_bytes: ReadableBuffer) -> str: ...
def find_spec(name: str, package: str | None = ...) -> importlib.machinery.ModuleSpec | None: ...
def spec_from_loader(
name: str, loader: importlib.abc.Loader | None, *, origin: str | None = ..., is_package: bool | None = ...
Expand All @@ -37,4 +37,4 @@ class LazyLoader(importlib.abc.Loader):
def factory(cls, loader: importlib.abc.Loader) -> Callable[..., LazyLoader]: ...
def exec_module(self, module: types.ModuleType) -> None: ...

def source_hash(source_bytes: bytes) -> int: ...
def source_hash(source_bytes: ReadableBuffer) -> int: ...
3 changes: 1 addition & 2 deletions stubs/pyinstaller/PyInstaller/compat.pyi
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
# https://pyinstaller.org/en/stable/hooks.html#module-PyInstaller.compat
from _typeshed import FileDescriptor, GenericPath, StrOrBytesPath
from collections.abc import Iterable
from importlib.abc import _Path
from types import ModuleType
from typing import AnyStr, overload
from typing_extensions import Literal, TypeAlias
Expand Down Expand Up @@ -69,7 +68,7 @@ def exec_python(*args: str, **kwargs: str | None) -> str: ...
def exec_python_rc(*args: str, **kwargs: str | None) -> int: ...
def expand_path(path: GenericPath[AnyStr]) -> AnyStr: ...
def getsitepackages(prefixes: Iterable[str] | None = ...) -> list[str]: ...
def importlib_load_source(name: str, pathname: _Path) -> ModuleType: ...
def importlib_load_source(name: str, pathname: str) -> ModuleType: ...

PY3_BASE_MODULES: set[str]
PURE_PYTHON_MODULE_TYPES: set[str]
Expand Down