From f5da6ab92af8ab952f0b9eee73d398702d7111b2 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 26 Aug 2024 17:57:44 +0200 Subject: [PATCH 1/4] wip: pyi import machinery --- pdoc/doc_pyi.py | 39 +++++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/pdoc/doc_pyi.py b/pdoc/doc_pyi.py index caf228b5..d8d14ea3 100644 --- a/pdoc/doc_pyi.py +++ b/pdoc/doc_pyi.py @@ -6,6 +6,7 @@ from __future__ import annotations +import importlib.util from pathlib import Path import sys import traceback @@ -45,13 +46,39 @@ def find_stub_file(module_name: str) -> Path | None: def _import_stub_file(module_name: str, stub_file: Path) -> types.ModuleType: - """Import the type stub outside of the normal import machinery.""" - code = compile(stub_file.read_text(), str(stub_file), "exec") - m = types.ModuleType(module_name) - m.__file__ = str(stub_file) - eval(code, m.__dict__, m.__dict__) + """ + Import the type stub outside of the normal import machinery. + + Note that currently, for objects imported by the stub file, the _original_ module + is used and not the corresponding stub file. + """ + sys.path_hooks.insert( + 0, + importlib.machinery.FileFinder.path_hook((importlib.machinery.SourceFileLoader, ['.pyi'])) + ) - return m + mods = {} + for k in list(sys.modules): + if k.startswith(module_name): + print("removing", k) + mods[k] = sys.modules.pop(k) + + importlib.invalidate_caches() + sys.path_importer_cache.clear() + + try: + loader = importlib.machinery.SourceFileLoader(module_name, str(stub_file)) + spec = importlib.util.spec_from_file_location(module_name, stub_file, loader=loader) + m = importlib.util.module_from_spec(spec) + loader.exec_module(m) + return m + finally: + sys.path_hooks.pop(0) + for k in list(sys.modules): + if k.startswith(module_name): + sys.modules.pop(k) + for k, v in mods.items(): + sys.modules[k] = v def _prepare_module(ns: doc.Namespace) -> None: From 720521d01eae137eacae1351a57a94957b3fd09e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 26 Aug 2024 18:00:58 +0200 Subject: [PATCH 2/4] improve pyi file importing --- CHANGELOG.md | 1 + pdoc/doc_pyi.py | 20 +-- test/test_snapshot.py | 2 +- .../{type_stub.html => type_stubs.html} | 165 +++++++++++------- test/testdata/type_stubs.txt | 19 ++ .../{type_stub.py => type_stubs/__init__.py} | 8 + .../__init__.pyi} | 1 + test/testdata/type_stubs/_utils.py | 2 + test/testdata/type_stubs/_utils.pyi | 2 + 9 files changed, 136 insertions(+), 84 deletions(-) rename test/testdata/{type_stub.html => type_stubs.html} (89%) create mode 100644 test/testdata/type_stubs.txt rename test/testdata/{type_stub.py => type_stubs/__init__.py} (83%) rename test/testdata/{type_stub.pyi => type_stubs/__init__.pyi} (93%) create mode 100644 test/testdata/type_stubs/_utils.py create mode 100644 test/testdata/type_stubs/_utils.pyi diff --git a/CHANGELOG.md b/CHANGELOG.md index c52ee93d..aae4934b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Fix a bug where entire modules would be excluded by `--no-include-undocumented`. To exclude modules, see https://pdoc.dev/docs/pdoc.html#exclude-submodules-from-being-documented. ([#728](https://github.com/mitmproxy/pdoc/pull/728), @mhils) +- Fix a bug where pdoc would crash when importing pyi files. - Fix a bug where subclasses of TypedDict subclasses would not render correctly. ([#729](https://github.com/mitmproxy/pdoc/pull/729), @mhils) diff --git a/pdoc/doc_pyi.py b/pdoc/doc_pyi.py index d8d14ea3..33a66eb8 100644 --- a/pdoc/doc_pyi.py +++ b/pdoc/doc_pyi.py @@ -52,20 +52,9 @@ def _import_stub_file(module_name: str, stub_file: Path) -> types.ModuleType: Note that currently, for objects imported by the stub file, the _original_ module is used and not the corresponding stub file. """ - sys.path_hooks.insert( - 0, + sys.path_hooks.append( importlib.machinery.FileFinder.path_hook((importlib.machinery.SourceFileLoader, ['.pyi'])) ) - - mods = {} - for k in list(sys.modules): - if k.startswith(module_name): - print("removing", k) - mods[k] = sys.modules.pop(k) - - importlib.invalidate_caches() - sys.path_importer_cache.clear() - try: loader = importlib.machinery.SourceFileLoader(module_name, str(stub_file)) spec = importlib.util.spec_from_file_location(module_name, stub_file, loader=loader) @@ -73,12 +62,7 @@ def _import_stub_file(module_name: str, stub_file: Path) -> types.ModuleType: loader.exec_module(m) return m finally: - sys.path_hooks.pop(0) - for k in list(sys.modules): - if k.startswith(module_name): - sys.modules.pop(k) - for k, v in mods.items(): - sys.modules[k] = v + sys.path_hooks.pop() def _prepare_module(ns: doc.Namespace) -> None: diff --git a/test/test_snapshot.py b/test/test_snapshot.py index d2d538b7..04bd8e56 100755 --- a/test/test_snapshot.py +++ b/test/test_snapshot.py @@ -164,7 +164,7 @@ def outfile(self, format: str) -> Path: Snapshot("pyo3_sample_library", specs=["pdoc_pyo3_sample_library"]), Snapshot("top_level_reimports", ["top_level_reimports"]), Snapshot("type_checking_imports", ["type_checking_imports.main"]), - Snapshot("type_stub", min_version=(3, 10)), + Snapshot("type_stubs", ["type_stubs"], min_version=(3, 10)), Snapshot( "visibility", render_options={ diff --git a/test/testdata/type_stub.html b/test/testdata/type_stubs.html similarity index 89% rename from test/testdata/type_stub.html rename to test/testdata/type_stubs.html index eea25d23..124d0547 100644 --- a/test/testdata/type_stub.html +++ b/test/testdata/type_stubs.html @@ -4,7 +4,7 @@ - type_stub API documentation + type_stubs API documentation @@ -59,6 +59,12 @@

API Documentation

+
  • + ImportedClass +
      +
    + +
    • @@ -73,47 +79,55 @@

    API Documentation

    -type_stub

    +type_stubs

    This module has an accompanying .pyi file with type stubs.

    - + - + 
     1"""
  2This module has an accompanying .pyi file with type stubs.
  3"""
- 4
+ 4from ._utils import ImportedClass
  5
- 6def func(x, y):
- 7    """A simple function."""
- 8
+ 6
+ 7def func(x, y):
+ 8    """A simple function."""
  9
-10var = []
-11"""A simple variable."""
-12
+10
+11var = []
+12"""A simple variable."""
 13
-14class Class:
-15    attr = 42
-16    """An attribute"""
-17
-18    def meth(self, y):
-19        """A simple method."""
-20
-21    class Subclass:
-22        attr = "42"
-23        """An attribute"""
-24
-25        def meth(self, y):
-26            """A simple method."""
-27
-28    def no_type_annotation(self, z):
-29        """A method not present in the .pyi file."""
-30
-31    def overloaded(self, x):
-32        """An overloaded method."""
+14
+15class Class:
+16    attr = 42
+17    """An attribute"""
+18
+19    def meth(self, y):
+20        """A simple method."""
+21
+22    class Subclass:
+23        attr = "42"
+24        """An attribute"""
+25
+26        def meth(self, y):
+27            """A simple method."""
+28
+29    def no_type_annotation(self, z):
+30        """A method not present in the .pyi file."""
+31
+32    def overloaded(self, x):
+33        """An overloaded method."""
+34
+35__all__ = [
+36    "func",
+37    "var",
+38    "Class",
+39    "ImportedClass",
+40]
    @@ -129,8 +143,8 @@

    - 
    7def func(x, y):
-8    """A simple function."""
+            
    8def func(x, y):
+9    """A simple function."""
 
    
 
 
@@ -164,25 +178,25 @@
    - 
    15class Class:
-16    attr = 42
-17    """An attribute"""
-18
-19    def meth(self, y):
-20        """A simple method."""
-21
-22    class Subclass:
-23        attr = "42"
-24        """An attribute"""
-25
-26        def meth(self, y):
-27            """A simple method."""
-28
-29    def no_type_annotation(self, z):
-30        """A method not present in the .pyi file."""
-31
-32    def overloaded(self, x):
-33        """An overloaded method."""
+            
    16class Class:
+17    attr = 42
+18    """An attribute"""
+19
+20    def meth(self, y):
+21        """A simple method."""
+22
+23    class Subclass:
+24        attr = "42"
+25        """An attribute"""
+26
+27        def meth(self, y):
+28            """A simple method."""
+29
+30    def no_type_annotation(self, z):
+31        """A method not present in the .pyi file."""
+32
+33    def overloaded(self, x):
+34        """An overloaded method."""
 
    
 
 
@@ -213,8 +227,8 @@
    - 
    19    def meth(self, y):
-20        """A simple method."""
+            
    20    def meth(self, y):
+21        """A simple method."""
 
    
 
 
@@ -234,8 +248,8 @@
    - 
    29    def no_type_annotation(self, z):
-30        """A method not present in the .pyi file."""
+            
    30    def no_type_annotation(self, z):
+31        """A method not present in the .pyi file."""
 
    
 
 
@@ -255,8 +269,8 @@
    - 
    32    def overloaded(self, x):
-33        """An overloaded method."""
+            
    33    def overloaded(self, x):
+34        """An overloaded method."""
 
    
 
 
@@ -277,12 +291,12 @@
    - 
    22    class Subclass:
-23        attr = "42"
-24        """An attribute"""
-25
-26        def meth(self, y):
-27            """A simple method."""
+            
    23    class Subclass:
+24        attr = "42"
+25        """An attribute"""
+26
+27        def meth(self, y):
+28            """A simple method."""
 
    
 
 
@@ -313,8 +327,8 @@
    - 
    26        def meth(self, y):
-27            """A simple method."""
+            
    27        def meth(self, y):
+28            """A simple method."""
 
    
 
 
@@ -324,6 +338,27 @@

    +
    + +
    + + class + ImportedClass: + + + +
    + + 
    2class ImportedClass:
+3    """Docstring from imported py file - ideally this should be overridden."""
+
    + + +

    Docstring from imported py file - ideally this should be overridden.

    +
    + + +
    \ No newline at end of file diff --git a/test/testdata/type_stubs.txt b/test/testdata/type_stubs.txt new file mode 100644 index 00000000..1bdac830 --- /dev/null +++ b/test/testdata/type_stubs.txt @@ -0,0 +1,19 @@ + int: ... # A simple function.> + + + + bool: ... # A simple method.> + + + bool: ... # A simple method.> + > + + + > + + > +> \ No newline at end of file diff --git a/test/testdata/type_stub.py b/test/testdata/type_stubs/__init__.py similarity index 83% rename from test/testdata/type_stub.py rename to test/testdata/type_stubs/__init__.py index e9492dba..ed00e07e 100644 --- a/test/testdata/type_stub.py +++ b/test/testdata/type_stubs/__init__.py @@ -1,6 +1,7 @@ """ This module has an accompanying .pyi file with type stubs. """ +from ._utils import ImportedClass def func(x, y): @@ -30,3 +31,10 @@ def no_type_annotation(self, z): def overloaded(self, x): """An overloaded method.""" + +__all__ = [ + "func", + "var", + "Class", + "ImportedClass", +] \ No newline at end of file diff --git a/test/testdata/type_stub.pyi b/test/testdata/type_stubs/__init__.pyi similarity index 93% rename from test/testdata/type_stub.pyi rename to test/testdata/type_stubs/__init__.pyi index fc0f403a..442d6f54 100644 --- a/test/testdata/type_stub.pyi +++ b/test/testdata/type_stubs/__init__.pyi @@ -1,6 +1,7 @@ from typing import Any from typing import Iterable from typing import overload +from ._utils import ImportedClass def func(x: str, y: Any, z: "Iterable[str]") -> int: ... diff --git a/test/testdata/type_stubs/_utils.py b/test/testdata/type_stubs/_utils.py new file mode 100644 index 00000000..0794c820 --- /dev/null +++ b/test/testdata/type_stubs/_utils.py @@ -0,0 +1,2 @@ +class ImportedClass: + """Docstring from imported py file - ideally this should be overridden.""" diff --git a/test/testdata/type_stubs/_utils.pyi b/test/testdata/type_stubs/_utils.pyi new file mode 100644 index 00000000..6eb8bccd --- /dev/null +++ b/test/testdata/type_stubs/_utils.pyi @@ -0,0 +1,2 @@ +class ImportedClass: + """Docstring from imported pyi file""" From be98a83df9d5c8620023f373bfe9be8e16afc244 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 27 Aug 2024 08:15:32 +0200 Subject: [PATCH 3/4] fixups --- pdoc/doc_pyi.py | 1 + test/testdata/type_stubs/__init__.pyi | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/pdoc/doc_pyi.py b/pdoc/doc_pyi.py index 33a66eb8..7e180ae7 100644 --- a/pdoc/doc_pyi.py +++ b/pdoc/doc_pyi.py @@ -58,6 +58,7 @@ def _import_stub_file(module_name: str, stub_file: Path) -> types.ModuleType: try: loader = importlib.machinery.SourceFileLoader(module_name, str(stub_file)) spec = importlib.util.spec_from_file_location(module_name, stub_file, loader=loader) + assert spec is not None m = importlib.util.module_from_spec(spec) loader.exec_module(m) return m diff --git a/test/testdata/type_stubs/__init__.pyi b/test/testdata/type_stubs/__init__.pyi index 442d6f54..d4ce3e6c 100644 --- a/test/testdata/type_stubs/__init__.pyi +++ b/test/testdata/type_stubs/__init__.pyi @@ -1,6 +1,7 @@ from typing import Any from typing import Iterable from typing import overload + from ._utils import ImportedClass def func(x: str, y: Any, z: "Iterable[str]") -> int: ... @@ -22,3 +23,10 @@ class Class: def overloaded(self, x: int) -> int: ... @overload def overloaded(self, x: str) -> str: ... + +__all__ = [ + "func", + "var", + "Class", + "ImportedClass", +] From 228655537ee3d0f6766fbfcdef5e758a12342332 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Tue, 27 Aug 2024 06:16:29 +0000 Subject: [PATCH 4/4] [autofix.ci] apply automated fixes --- CHANGELOG.md | 1 + pdoc/doc_pyi.py | 10 +- test/testdata/type_stubs.html | 140 ++++++++++++++------------- test/testdata/type_stubs/__init__.py | 4 +- 4 files changed, 82 insertions(+), 73 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aae4934b..da93e31e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ To exclude modules, see https://pdoc.dev/docs/pdoc.html#exclude-submodules-from-being-documented. ([#728](https://github.com/mitmproxy/pdoc/pull/728), @mhils) - Fix a bug where pdoc would crash when importing pyi files. + ([#732](https://github.com/mitmproxy/pdoc/pull/732), @mhils) - Fix a bug where subclasses of TypedDict subclasses would not render correctly. ([#729](https://github.com/mitmproxy/pdoc/pull/729), @mhils) diff --git a/pdoc/doc_pyi.py b/pdoc/doc_pyi.py index 7e180ae7..f1df53b4 100644 --- a/pdoc/doc_pyi.py +++ b/pdoc/doc_pyi.py @@ -48,16 +48,20 @@ def find_stub_file(module_name: str) -> Path | None: def _import_stub_file(module_name: str, stub_file: Path) -> types.ModuleType: """ Import the type stub outside of the normal import machinery. - + Note that currently, for objects imported by the stub file, the _original_ module is used and not the corresponding stub file. """ sys.path_hooks.append( - importlib.machinery.FileFinder.path_hook((importlib.machinery.SourceFileLoader, ['.pyi'])) + importlib.machinery.FileFinder.path_hook( + (importlib.machinery.SourceFileLoader, [".pyi"]) + ) ) try: loader = importlib.machinery.SourceFileLoader(module_name, str(stub_file)) - spec = importlib.util.spec_from_file_location(module_name, stub_file, loader=loader) + spec = importlib.util.spec_from_file_location( + module_name, stub_file, loader=loader + ) assert spec is not None m = importlib.util.module_from_spec(spec) loader.exec_module(m) diff --git a/test/testdata/type_stubs.html b/test/testdata/type_stubs.html index 124d0547..e32a54d3 100644 --- a/test/testdata/type_stubs.html +++ b/test/testdata/type_stubs.html @@ -91,43 +91,45 @@ 

     1"""
  2This module has an accompanying .pyi file with type stubs.
  3"""
- 4from ._utils import ImportedClass
- 5
+ 4
+ 5from ._utils import ImportedClass
  6
- 7def func(x, y):
- 8    """A simple function."""
- 9
+ 7
+ 8def func(x, y):
+ 9    """A simple function."""
 10
-11var = []
-12"""A simple variable."""
-13
+11
+12var = []
+13"""A simple variable."""
 14
-15class Class:
-16    attr = 42
-17    """An attribute"""
-18
-19    def meth(self, y):
-20        """A simple method."""
-21
-22    class Subclass:
-23        attr = "42"
-24        """An attribute"""
-25
-26        def meth(self, y):
-27            """A simple method."""
-28
-29    def no_type_annotation(self, z):
-30        """A method not present in the .pyi file."""
-31
-32    def overloaded(self, x):
-33        """An overloaded method."""
-34
-35__all__ = [
-36    "func",
-37    "var",
-38    "Class",
-39    "ImportedClass",
-40]
+15
+16class Class:
+17    attr = 42
+18    """An attribute"""
+19
+20    def meth(self, y):
+21        """A simple method."""
+22
+23    class Subclass:
+24        attr = "42"
+25        """An attribute"""
+26
+27        def meth(self, y):
+28            """A simple method."""
+29
+30    def no_type_annotation(self, z):
+31        """A method not present in the .pyi file."""
+32
+33    def overloaded(self, x):
+34        """An overloaded method."""
+35
+36
+37__all__ = [
+38    "func",
+39    "var",
+40    "Class",
+41    "ImportedClass",
+42]
    @@ -143,8 +145,8 @@

    - 
    8def func(x, y):
-9    """A simple function."""
+            
     9def func(x, y):
+10    """A simple function."""
 
    
 
 
@@ -178,25 +180,25 @@
    - 
    16class Class:
-17    attr = 42
-18    """An attribute"""
-19
-20    def meth(self, y):
-21        """A simple method."""
-22
-23    class Subclass:
-24        attr = "42"
-25        """An attribute"""
-26
-27        def meth(self, y):
-28            """A simple method."""
-29
-30    def no_type_annotation(self, z):
-31        """A method not present in the .pyi file."""
-32
-33    def overloaded(self, x):
-34        """An overloaded method."""
+            
    17class Class:
+18    attr = 42
+19    """An attribute"""
+20
+21    def meth(self, y):
+22        """A simple method."""
+23
+24    class Subclass:
+25        attr = "42"
+26        """An attribute"""
+27
+28        def meth(self, y):
+29            """A simple method."""
+30
+31    def no_type_annotation(self, z):
+32        """A method not present in the .pyi file."""
+33
+34    def overloaded(self, x):
+35        """An overloaded method."""
 
    
 
 
@@ -227,8 +229,8 @@
    - 
    20    def meth(self, y):
-21        """A simple method."""
+            
    21    def meth(self, y):
+22        """A simple method."""
 
    
 
 
@@ -248,8 +250,8 @@
    - 
    30    def no_type_annotation(self, z):
-31        """A method not present in the .pyi file."""
+            
    31    def no_type_annotation(self, z):
+32        """A method not present in the .pyi file."""
 
    
 
 
@@ -269,8 +271,8 @@
    - 
    33    def overloaded(self, x):
-34        """An overloaded method."""
+            
    34    def overloaded(self, x):
+35        """An overloaded method."""
 
    
 
 
@@ -291,12 +293,12 @@
    - 
    23    class Subclass:
-24        attr = "42"
-25        """An attribute"""
-26
-27        def meth(self, y):
-28            """A simple method."""
+            
    24    class Subclass:
+25        attr = "42"
+26        """An attribute"""
+27
+28        def meth(self, y):
+29            """A simple method."""
 
    
 
 
@@ -327,8 +329,8 @@
    - 
    27        def meth(self, y):
-28            """A simple method."""
+            
    28        def meth(self, y):
+29            """A simple method."""
 
    
 
 
diff --git a/test/testdata/type_stubs/__init__.py b/test/testdata/type_stubs/__init__.py
index ed00e07e..82234923 100644
--- a/test/testdata/type_stubs/__init__.py
+++ b/test/testdata/type_stubs/__init__.py
@@ -1,6 +1,7 @@
 """
 This module has an accompanying .pyi file with type stubs.
 """
+
 from ._utils import ImportedClass
 
 
@@ -32,9 +33,10 @@ def no_type_annotation(self, z):
     def overloaded(self, x):
         """An overloaded method."""
 
+
 __all__ = [
     "func",
     "var",
     "Class",
     "ImportedClass",
-]
\ No newline at end of file
+]