From 4c2518245e831446678a231b93d0d9c25b0e6090 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sun, 19 Jan 2025 01:04:25 +0000 Subject: [PATCH 1/9] Refactor ``import_module`` --- sphinx/ext/autodoc/importer.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/sphinx/ext/autodoc/importer.py b/sphinx/ext/autodoc/importer.py index ab12df51097..c6e761800b4 100644 --- a/sphinx/ext/autodoc/importer.py +++ b/sphinx/ext/autodoc/importer.py @@ -9,6 +9,7 @@ import traceback import typing from enum import Enum +from importlib.util import find_spec, module_from_spec from typing import TYPE_CHECKING, NamedTuple from sphinx.errors import PycodeError @@ -151,9 +152,19 @@ def unmangle(subject: Any, name: str) -> str | None: def import_module(modname: str) -> Any: - """Call importlib.import_module(modname), convert exceptions to ImportError.""" try: - return importlib.import_module(modname) + spec = find_spec(modname) + if spec is None: + msg = f'No module named {modname!r}' + raise ModuleNotFoundError(msg, name=modname) # NoQA: TRY301 + if spec.loader is None: + msg = 'missing loader' + raise ImportError(msg, name=spec.name) # NoQA: TRY301 + sys.modules[modname] = module = module_from_spec(spec) + spec.loader.exec_module(module) + return module + except ImportError: + raise except BaseException as exc: # Importing modules may cause any side effects, including # SystemExit, so we need to catch all errors. @@ -249,7 +260,8 @@ def import_object( if isinstance(exc, ImportError): # import_module() raises ImportError having real exception obj and # traceback - real_exc, traceback_msg = exc.args + real_exc = exc.args[0] + traceback_msg = traceback.format_exception(exc) if isinstance(real_exc, SystemExit): errmsg += ( '; the module executes module level statement ' From d91b175c7e0efc63f42f6ecccea67b83a7811b4d Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sun, 19 Jan 2025 01:07:08 +0000 Subject: [PATCH 2/9] Move SPHINX_AUTODOC_RELOAD_MODULES to ``import_module`` --- sphinx/ext/autodoc/importer.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/sphinx/ext/autodoc/importer.py b/sphinx/ext/autodoc/importer.py index c6e761800b4..42ddef16671 100644 --- a/sphinx/ext/autodoc/importer.py +++ b/sphinx/ext/autodoc/importer.py @@ -152,6 +152,7 @@ def unmangle(subject: Any, name: str) -> str | None: def import_module(modname: str) -> Any: + original_module_names = frozenset(sys.modules) try: spec = find_spec(modname) if spec is None: @@ -162,13 +163,25 @@ def import_module(modname: str) -> Any: raise ImportError(msg, name=spec.name) # NoQA: TRY301 sys.modules[modname] = module = module_from_spec(spec) spec.loader.exec_module(module) - return module except ImportError: raise except BaseException as exc: # Importing modules may cause any side effects, including # SystemExit, so we need to catch all errors. raise ImportError(exc, traceback.format_exc()) from exc + if os.environ.get('SPHINX_AUTODOC_RELOAD_MODULES'): + new_modules = [m for m in sys.modules if m not in original_module_names] + # Try reloading modules with ``typing.TYPE_CHECKING == True``. + try: + typing.TYPE_CHECKING = True + # Ignore failures; we've already successfully loaded these modules + with contextlib.suppress(ImportError, KeyError): + for m in new_modules: + _reload_module(sys.modules[m]) + finally: + typing.TYPE_CHECKING = False + module = sys.modules[modname] + return module def _reload_module(module: ModuleType) -> Any: @@ -198,22 +211,7 @@ def import_object( objpath = objpath.copy() while module is None: try: - original_module_names = frozenset(sys.modules) module = import_module(modname) - if os.environ.get('SPHINX_AUTODOC_RELOAD_MODULES'): - new_modules = [ - m for m in sys.modules if m not in original_module_names - ] - # Try reloading modules with ``typing.TYPE_CHECKING == True``. - try: - typing.TYPE_CHECKING = True - # Ignore failures; we've already successfully loaded these modules - with contextlib.suppress(ImportError, KeyError): - for m in new_modules: - _reload_module(sys.modules[m]) - finally: - typing.TYPE_CHECKING = False - module = sys.modules[modname] logger.debug('[autodoc] import %s => %r', modname, module) except ImportError as exc: logger.debug('[autodoc] import %s => failed', modname) From b93798f7ba1b5a9c5420671b47637df0213d1c2f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sun, 19 Jan 2025 01:17:43 +0000 Subject: [PATCH 3/9] Find '.pyi' files for native modules --- sphinx/ext/autodoc/importer.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/sphinx/ext/autodoc/importer.py b/sphinx/ext/autodoc/importer.py index 42ddef16671..30cb0e50e4c 100644 --- a/sphinx/ext/autodoc/importer.py +++ b/sphinx/ext/autodoc/importer.py @@ -9,7 +9,9 @@ import traceback import typing from enum import Enum -from importlib.util import find_spec, module_from_spec +from importlib.machinery import EXTENSION_SUFFIXES +from importlib.util import find_spec, module_from_spec, spec_from_file_location +from pathlib import Path from typing import TYPE_CHECKING, NamedTuple from sphinx.errors import PycodeError @@ -33,6 +35,7 @@ from sphinx.ext.autodoc import ObjectMember +_NATIVE_SUFFIXES: frozenset[str] = frozenset({'.pyx', *EXTENSION_SUFFIXES}) logger = logging.getLogger(__name__) @@ -158,6 +161,18 @@ def import_module(modname: str) -> Any: if spec is None: msg = f'No module named {modname!r}' raise ModuleNotFoundError(msg, name=modname) # NoQA: TRY301 + if spec.origin is not None: + # Try finding a spec for a '.pyi' file for native modules. + for suffix in _NATIVE_SUFFIXES: + if not spec.origin.endswith(suffix): + continue + pyi_path = Path(spec.origin.removesuffix(suffix) + '.pyi') + if not pyi_path.is_file(): + continue + pyi_spec = spec_from_file_location(modname, pyi_path) + if pyi_spec is not None: + spec = pyi_spec + break if spec.loader is None: msg = 'missing loader' raise ImportError(msg, name=spec.name) # NoQA: TRY301 From f70714ae6382cc2fb6cd80e78345f8adbc146fc7 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sun, 19 Jan 2025 02:00:46 +0000 Subject: [PATCH 4/9] Preferentially load stub files to native modules --- CHANGES.rst | 3 +++ sphinx/ext/autodoc/importer.py | 27 ++++++++++++++++--- .../fish_licence/halibut.pyd | 0 .../fish_licence/halibut.pyi | 0 .../test_ext_autodoc_importer.py | 18 +++++++++++++ 5 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 tests/roots/test-ext-apidoc-duplicates/fish_licence/halibut.pyd create mode 100644 tests/roots/test-ext-apidoc-duplicates/fish_licence/halibut.pyi create mode 100644 tests/test_extensions/test_ext_autodoc_importer.py diff --git a/CHANGES.rst b/CHANGES.rst index 69c4ada5cc1..ff344d06d41 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -50,6 +50,9 @@ Features added pattern doesn't match any documents, via the new ``toc.glob_not_matching`` warning sub-type. Patch by Slawek Figiel. +* #7630, #4824: autodoc: Use :file:`.pyi` type stub files + to auto-document native modules. + Patch by Adam Turner, partially based on work by Allie Fitter. Bugs fixed ---------- diff --git a/sphinx/ext/autodoc/importer.py b/sphinx/ext/autodoc/importer.py index 30cb0e50e4c..2d30f4f62da 100644 --- a/sphinx/ext/autodoc/importer.py +++ b/sphinx/ext/autodoc/importer.py @@ -9,8 +9,9 @@ import traceback import typing from enum import Enum +from importlib.abc import FileLoader from importlib.machinery import EXTENSION_SUFFIXES -from importlib.util import find_spec, module_from_spec, spec_from_file_location +from importlib.util import decode_source, find_spec, module_from_spec, spec_from_loader from pathlib import Path from typing import TYPE_CHECKING, NamedTuple @@ -162,14 +163,15 @@ def import_module(modname: str) -> Any: msg = f'No module named {modname!r}' raise ModuleNotFoundError(msg, name=modname) # NoQA: TRY301 if spec.origin is not None: - # Try finding a spec for a '.pyi' file for native modules. + # Try finding a spec for a '.pyi' stubs file for native modules. for suffix in _NATIVE_SUFFIXES: if not spec.origin.endswith(suffix): continue pyi_path = Path(spec.origin.removesuffix(suffix) + '.pyi') if not pyi_path.is_file(): continue - pyi_spec = spec_from_file_location(modname, pyi_path) + pyi_loader = _StubFileLoader(modname, path=str(pyi_path)) + pyi_spec = spec_from_loader(modname, loader=pyi_loader) if pyi_spec is not None: spec = pyi_spec break @@ -192,6 +194,9 @@ def import_module(modname: str) -> Any: # Ignore failures; we've already successfully loaded these modules with contextlib.suppress(ImportError, KeyError): for m in new_modules: + mod_path = getattr(sys.modules[m], '__file__', '') + if mod_path and mod_path.endswith('.pyi'): + continue _reload_module(sys.modules[m]) finally: typing.TYPE_CHECKING = False @@ -199,6 +204,22 @@ def import_module(modname: str) -> Any: return module +class _StubFileLoader(FileLoader): + """Load modules from ``.pyi`` stub files.""" + + def get_source(self, fullname: str) -> str: + path = self.get_filename(fullname) + for suffix in _NATIVE_SUFFIXES: + if not path.endswith(suffix): + continue + path = path.removesuffix(suffix) + '.pyi' + try: + source_bytes = self.get_data(path) + except OSError as exc: + raise ImportError from exc + return decode_source(source_bytes) + + def _reload_module(module: ModuleType) -> Any: """Call importlib.reload(module), convert exceptions to ImportError""" try: diff --git a/tests/roots/test-ext-apidoc-duplicates/fish_licence/halibut.pyd b/tests/roots/test-ext-apidoc-duplicates/fish_licence/halibut.pyd new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/roots/test-ext-apidoc-duplicates/fish_licence/halibut.pyi b/tests/roots/test-ext-apidoc-duplicates/fish_licence/halibut.pyi new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_extensions/test_ext_autodoc_importer.py b/tests/test_extensions/test_ext_autodoc_importer.py new file mode 100644 index 00000000000..d1933e4ecad --- /dev/null +++ b/tests/test_extensions/test_ext_autodoc_importer.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING + +from sphinx.ext.autodoc.importer import import_module + +if TYPE_CHECKING: + from pathlib import Path + + +def test_import_native_module_stubs(rootdir: Path) -> None: + sys_path = list(sys.path) + sys.path.insert(0, str(rootdir / 'test-ext-apidoc-duplicates')) + halibut = import_module('fish_licence.halibut') + assert halibut.__file__.endswith('halibut.pyi') + assert halibut.__spec__.origin.endswith('halibut.pyi') + sys.path[:] = sys_path From 4f7b4f81105d6a7ffb5342947207ddcdc54d42b1 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sun, 19 Jan 2025 02:11:28 +0000 Subject: [PATCH 5/9] Refactor ``import_module`` --- sphinx/ext/autodoc/importer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sphinx/ext/autodoc/importer.py b/sphinx/ext/autodoc/importer.py index 2d30f4f62da..c5cb9970513 100644 --- a/sphinx/ext/autodoc/importer.py +++ b/sphinx/ext/autodoc/importer.py @@ -155,7 +155,7 @@ def unmangle(subject: Any, name: str) -> str | None: return name -def import_module(modname: str) -> Any: +def import_module(modname: str, try_reload: bool = False) -> Any: original_module_names = frozenset(sys.modules) try: spec = find_spec(modname) @@ -186,7 +186,7 @@ def import_module(modname: str) -> Any: # Importing modules may cause any side effects, including # SystemExit, so we need to catch all errors. raise ImportError(exc, traceback.format_exc()) from exc - if os.environ.get('SPHINX_AUTODOC_RELOAD_MODULES'): + if try_reload and os.environ.get('SPHINX_AUTODOC_RELOAD_MODULES'): new_modules = [m for m in sys.modules if m not in original_module_names] # Try reloading modules with ``typing.TYPE_CHECKING == True``. try: @@ -247,7 +247,7 @@ def import_object( objpath = objpath.copy() while module is None: try: - module = import_module(modname) + module = import_module(modname, try_reload=True) logger.debug('[autodoc] import %s => %r', modname, module) except ImportError as exc: logger.debug('[autodoc] import %s => failed', modname) From d5f4a97771cf205d3a0070676431f97b8f283007 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sun, 19 Jan 2025 02:25:52 +0000 Subject: [PATCH 6/9] return cached module --- sphinx/ext/autodoc/importer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sphinx/ext/autodoc/importer.py b/sphinx/ext/autodoc/importer.py index c5cb9970513..e1e371ced5d 100644 --- a/sphinx/ext/autodoc/importer.py +++ b/sphinx/ext/autodoc/importer.py @@ -156,6 +156,9 @@ def unmangle(subject: Any, name: str) -> str | None: def import_module(modname: str, try_reload: bool = False) -> Any: + if modname in sys.modules: + return sys.modules[modname] + original_module_names = frozenset(sys.modules) try: spec = find_spec(modname) From 17ed4303fc10a6d11c18d1e8d7e7fcec86d51999 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sun, 19 Jan 2025 02:34:39 +0000 Subject: [PATCH 7/9] fix tests --- sphinx/ext/autodoc/importer.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/sphinx/ext/autodoc/importer.py b/sphinx/ext/autodoc/importer.py index e1e371ced5d..c51a124cd5c 100644 --- a/sphinx/ext/autodoc/importer.py +++ b/sphinx/ext/autodoc/importer.py @@ -165,6 +165,7 @@ def import_module(modname: str, try_reload: bool = False) -> Any: if spec is None: msg = f'No module named {modname!r}' raise ModuleNotFoundError(msg, name=modname) # NoQA: TRY301 + pyi_path = None if spec.origin is not None: # Try finding a spec for a '.pyi' stubs file for native modules. for suffix in _NATIVE_SUFFIXES: @@ -178,11 +179,14 @@ def import_module(modname: str, try_reload: bool = False) -> Any: if pyi_spec is not None: spec = pyi_spec break - if spec.loader is None: - msg = 'missing loader' - raise ImportError(msg, name=spec.name) # NoQA: TRY301 - sys.modules[modname] = module = module_from_spec(spec) - spec.loader.exec_module(module) + if pyi_path is None: + module = importlib.import_module(modname) + else: + if spec.loader is None: + msg = 'missing loader' + raise ImportError(msg, name=spec.name) # NoQA: TRY301 + sys.modules[modname] = module = module_from_spec(spec) + spec.loader.exec_module(module) except ImportError: raise except BaseException as exc: From d0f4f55ffc4acee571672be5688007081e38d4d7 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Tue, 21 Jan 2025 19:24:10 +0000 Subject: [PATCH 8/9] Test path too --- tests/test_extensions/test_ext_autodoc_importer.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/test_extensions/test_ext_autodoc_importer.py b/tests/test_extensions/test_ext_autodoc_importer.py index d1933e4ecad..3f34c6ee279 100644 --- a/tests/test_extensions/test_ext_autodoc_importer.py +++ b/tests/test_extensions/test_ext_autodoc_importer.py @@ -10,9 +10,16 @@ def test_import_native_module_stubs(rootdir: Path) -> None: + fish_licence_root = rootdir / 'test-ext-apidoc-duplicates' + sys_path = list(sys.path) - sys.path.insert(0, str(rootdir / 'test-ext-apidoc-duplicates')) + sys.path.insert(0, str(fish_licence_root)) halibut = import_module('fish_licence.halibut') + sys.path[:] = sys_path + assert halibut.__file__.endswith('halibut.pyi') assert halibut.__spec__.origin.endswith('halibut.pyi') - sys.path[:] = sys_path + + halibut_path = Path(halibut.__file__).resolve() + assert halibut_path.is_file() + assert halibut_path == fish_licence_root / 'fish_licence' / 'halibut.pyi' From e57a7405a865761757b2eb87fa73a7d73d0b3da0 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Tue, 21 Jan 2025 19:28:41 +0000 Subject: [PATCH 9/9] Fix imports --- tests/test_extensions/test_ext_autodoc_importer.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_extensions/test_ext_autodoc_importer.py b/tests/test_extensions/test_ext_autodoc_importer.py index 3f34c6ee279..f14b8256c14 100644 --- a/tests/test_extensions/test_ext_autodoc_importer.py +++ b/tests/test_extensions/test_ext_autodoc_importer.py @@ -1,13 +1,10 @@ from __future__ import annotations import sys -from typing import TYPE_CHECKING +from pathlib import Path from sphinx.ext.autodoc.importer import import_module -if TYPE_CHECKING: - from pathlib import Path - def test_import_native_module_stubs(rootdir: Path) -> None: fish_licence_root = rootdir / 'test-ext-apidoc-duplicates'