diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 0000000..db24720 --- /dev/null +++ b/.codecov.yml @@ -0,0 +1 @@ +comment: off diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..dce33c1 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,4 @@ +[report] +exclude_lines = + pragma: no cover + raise NotImplementedError() diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3f5f974 --- /dev/null +++ b/.gitignore @@ -0,0 +1,88 @@ +*.py[cod] +.DS_Store +# C extensions +*.so + +# Packages +*.egg +*.egg-info +build +eggs +.eggs +parts +var +sdist +develop-eggs +.installed.cfg +lib +lib64 +MANIFEST + +# Installer logs +pip-log.txt +npm-debug.log +pip-selfcheck.json + +# Unit test / coverage reports +.coverage +.tox +nosetests.xml +htmlcov +.cache +.pytest_cache +.mypy_cache + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# SQLite +test_exp_framework + +# npm +node_modules/ + +# dolphin +.directory +libpeerconnection.log + +# setuptools +dist + +# IDE Files +atlassian-ide-plugin.xml +.idea/ +*.swp +*.kate-swp +.ropeproject/ + +# Python3 Venv Files +.venv/ +bin/ +include/ +lib/ +lib64 +pyvenv.cfg +share/ +venv/ +.python-version + +# Cython +*.c + +# Emacs backup +*~ + +# VSCode +/.vscode + +# Automatically generated files +docs/preconvert +site/ +out +poetry.lock + diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..f172bd0 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,57 @@ +sudo: false +language: python + +branches: + except: + - requires-io-master + +env: + global: + - CI_DEPS=codecov>=2.0.5 + - CI_COMMANDS=codecov + +git: + depth: 10000 + +matrix: + fast_finish: true + include: + - python: 3.5 + env: TOXENV=py35 + sudo: required + - python: 3.6 + env: TOXENV=py36 + sudo: required + - python: 3.6 + env: TOXENV=lint + - python: "3.7-dev" + env: TOXENV=py37 + +install: + - pip install tox virtualenv setuptools + +script: + # All these steps MUST succeed for the job to be successful! + # Using the after_success block DOES NOT capture errors. + # Pull requests might break our build - we need to check this. + # Pull requests are not allowed to upload build artifacts - enforced by cibuild script. + - | + if [[ $TRAVIS_COMMIT_MESSAGE = *"[notest]"* ]]; then + echo "!!!! Skipping tests." + else + tox -- --verbose --cov-report=term + fi + + +notifications: + slack: + -rooms: + mitmproxy:YaDGC9Gt9TEM7o8zkC2OLNsu + on_success: change + on_failure: change + on_start: never + +cache: + directories: + - $HOME/.pyenv + - $HOME/.cache/pip diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..44334e9 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,23 @@ +Install the latest +=================== + +To install the latest version of pdocs simply run: + +`pip3 install pdocs` + +OR + +`poetry add pdocs` + +OR + +`pipenv install pdocs` + +see the [Installation QuickStart](https://timothycrosley.github.io/portray/docs/quick_start/1.-installation/) for more instructions. + +Changelog +========= + +## 1.0.0 - TBD +Initial API stable release of pdocs + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..deb036b --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Timothy Crosley + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..ee8b936 --- /dev/null +++ b/README.md @@ -0,0 +1,99 @@ + +[![Build Status](https://travis-ci.org/mitmproxy/pdoc.svg?branch=master)](https://travis-ci.org/mitmproxy/pdoc) +[![PyPI Version](https://shields.mitmproxy.org/pypi/v/pdoc.svg)](https://pypi.org/project/pdoc/) + +`pdoc` is a library and a command line program to discover the public +interface of a Python module or package. The `pdoc` script can be used to +generate plain text or HTML of a module's public interface, or it can be used +to run an HTTP server that serves generated HTML for installed modules. + + +Installation +------------ + + pip install pdoc + + +Features +-------- + +* Support for documenting data representation by traversing the abstract syntax + to find docstrings for module, class and instance variables. +* For cases where docstrings aren't appropriate (like a + [namedtuple](http://docs.python.org/2.7/library/collections.html#namedtuple-factory-function-for-tuples-with-named-fields)), + the special variable `__pdoc__` can be used in your module to + document any identifier in your public interface. +* Usage is simple. Just write your documentation as Markdown. There are no + added special syntax rules. +* `pdoc` respects your `__all__` variable when present. +* `pdoc` will automatically link identifiers in your docstrings to its + corresponding documentation. +* When `pdoc` is run as an HTTP server, external linking is supported between + packages. +* The `pdoc` HTTP server will cache generated documentation and automatically + regenerate it if the source code has been updated. +* When available, source code for modules, functions and classes can be viewed + in the HTML documentation. +* Inheritance is used when possible to infer docstrings for class members. + +The above features are explained in more detail in pdoc's documentation. + +`pdoc` is compatible with Python 3.5 and newer. + + +Example usage +------------- +`pdoc` will accept a Python module file, package directory or an import path. +For example, to view the documentation for the `csv` module in the console: + + pdoc csv + +Or, you could view it by pointing at the file directly: + + pdoc /usr/lib/python3.7/csv.py + +Submodules are fine too: + + pdoc multiprocessing.pool + +You can also filter the documentation with a keyword: + + pdoc csv reader + +Generate HTML with the `--html` switch: + + pdoc --html csv + +A file called `csv.m.html` will be written to the current directory. + +Or start an HTTP server that shows documentation for any installed module: + + pdoc --http + +Then open your web browser to `http://localhost:8080`. + +There are many other options to explore. You can see them all by running: + + pdoc --help + + +Submodule loading +----------------- + +`pdoc` uses idiomatic Python when loading your modules. Therefore, for `pdoc` to +find any submodules of the input module you specify on the command line, those +modules must be available through Python's ordinary module loading process. + +This is not a problem for globally installed modules like `sys`, but can be a +problem for your own sub-modules depending on how you have installed them. + +To ensure that `pdoc` can load any submodules imported by the modules you are +generating documentation for, you should add the appropriate directories to your +`PYTHONPATH` environment variable. + +For example, if a local module `a.py` imports `b.py` that is installed as +`/home/jsmith/pylib/b.py`, then you should make sure that your `PYTHONPATH` +includes `/home/jsmith/pylib`. + +If `pdoc` cannot load any modules imported by the input module, it will exit +with an error message indicating which module could not be loaded. diff --git a/pdocs/__init__.py b/pdocs/__init__.py new file mode 100644 index 0000000..8302bf1 --- /dev/null +++ b/pdocs/__init__.py @@ -0,0 +1,178 @@ +""" +Module pdoc provides types and functions for accessing the public +documentation of a Python module. This includes modules (and +sub-modules), functions, classes and module, class and instance +variables. Docstrings are taken from modules, functions and classes +using the special `__doc__` attribute. Docstrings for variables are +extracted by examining the module's abstract syntax tree. + +The public interface of a module is determined through one of two +ways. If `__all__` is defined in the module, then all identifiers in +that list will be considered public. No other identifiers will be +considered as public. Conversely, if `__all__` is not defined, then +`pdoc` will heuristically determine the public interface. There are +three rules that are applied to each identifier in the module: + +1. If the name starts with an underscore, it is **not** public. + +2. If the name is defined in a different module, it is **not** public. + +3. If the name refers to an immediate sub-module, then it is public. + +Once documentation for a module is created with `pdoc.Module`, it +can be output as either HTML or plain text using the covenience +functions `pdoc.html` and `pdoc.text`, or the corresponding methods +`pdoc.Module.html` and `pdoc.Module.text`. + +Alternatively, you may run an HTTP server with the `pdoc` script +included with this module. + + +Compatibility +------------- +`pdoc` requires Python 3.6 or later. + + +Contributing +------------ +`pdoc` [is on GitHub](https://github.com/mitmproxy/pdoc). Pull +requests and bug reports are welcome. + + +Linking to other identifiers +---------------------------- +In your documentation, you may link to other identifiers in +your module or submodules. Linking is automatically done for +you whenever you surround an identifier with a back quote +(grave). The identifier name must be fully qualified. For +example, \`pdoc.Doc.docstring\` is correct while +\`Doc.docstring\` is incorrect. + +If the `pdoc` script is used to run an HTTP server, then external +linking to other packages installed is possible. No extra work is +necessary; simply use the fully qualified path. For example, +\`nflvid.slice\` will create a link to the `nflvid.slice` +function, which is **not** a part of `pdoc` at all. + + +Where does pdoc get documentation from? +--------------------------------------- +Broadly speaking, `pdoc` gets everything you see from introspecting the +module. This includes words describing a particular module, class, +function or variable. While `pdoc` does some analysis on the source +code of a module, importing the module itself is necessary to use +Python's introspection features. + +In Python, objects like modules, functions, classes and methods have +a special attribute named `__doc__` which contains that object's +*docstring*. The docstring comes from a special placement of a string +in your source code. For example, the following code shows how to +define a function with a docstring and access the contents of that +docstring: + + #!python + >>> def test(): + ... '''This is a docstring.''' + ... pass + ... + >>> test.__doc__ + 'This is a docstring.' + +Something similar can be done for classes and modules too. For classes, +the docstring should come on the line immediately following `class +...`. For modules, the docstring should start on the first line of +the file. These docstrings are what you see for each module, class, +function and method listed in the documentation produced by `pdoc`. + +The above just about covers *standard* uses of docstrings in Python. +`pdoc` extends the above in a few important ways. + + +### Special docstring conventions used by `pdoc` + +**Firstly**, docstrings can be inherited. Consider the following code +sample: + + #!python + >>> class A (object): + ... def test(): + ... '''Docstring for A.''' + ... + >>> class B (A): + ... def test(): + ... pass + ... + >>> print(A.test.__doc__) + Docstring for A. + >>> print(B.test.__doc__) + None + +In Python, the docstring for `B.test` is empty, even though one was +defined in `A.test`. If `pdoc` generates documentation for the above +code, then it will automatically attach the docstring for `A.test` to +`B.test` only if `B.test` does not have a docstring. In the default +HTML output, an inherited docstring is grey. + +**Secondly**, docstrings can be attached to variables, which includes +module (or global) variables, class variables and instance variables. +Python by itself [does not allow docstrings to be attached to +variables](http://www.python.org/dev/peps/pep-0224). For example: + + #!python + variable = "SomeValue" + '''Docstring for variable.''' + +The resulting `variable` will have no `__doc__` attribute. To +compensate, `pdoc` will read the source code when it's available to +infer a connection between a variable and a docstring. The connection +is only made when an assignment statement is followed by a docstring. + +Something similar is done for instance variables as well. By +convention, instance variables are initialized in a class's `__init__` +method. Therefore, `pdoc` adheres to that convention and looks for +docstrings of variables like so: + + #!python + def __init__(self): + self.variable = "SomeValue" + '''Docstring for instance variable.''' + +Note that `pdoc` only considers attributes defined on `self` as +instance variables. + +Class and instance variables can also have inherited docstrings. + +**Thirdly and finally**, docstrings can be overridden with a special +`__pdoc__` dictionary that `pdoc` inspects if it exists. The keys of +`__pdoc__` should be identifiers within the scope of the module. (In +the case of an instance variable `self.variable` for class `A`, its +module identifier would be `A.variable`.) The values of `__pdoc__` +should be docstrings. + +This particular feature is useful when there's no feasible way of +attaching a docstring to something. A good example of this is a +[namedtuple](http://goo.gl/akfXJ9): + + #!python + __pdoc__ = {} + + Table = namedtuple('Table', ['types', 'names', 'rows']) + __pdoc__['Table.types'] = 'Types for each column in the table.' + __pdoc__['Table.names'] = 'The names of each column in the table.' + __pdoc__['Table.rows'] = 'Lists corresponding to each row in the table.' + +`pdoc` will then show `Table` as a class with documentation for the +`types`, `names` and `rows` members. + +Note that assignments to `__pdoc__` need to placed where they'll be +executed when the module is imported. For example, at the top level +of a module or in the definition of a class. + +If `__pdoc__[key] = None`, then `key` will not be included in the +public interface of the module. +""" + +__version__ = "0.3.2" +""" +The current version of pdoc. This value is read from `setup.py`. +""" diff --git a/pdocs/cli.py b/pdocs/cli.py new file mode 100644 index 0000000..74d7b1c --- /dev/null +++ b/pdocs/cli.py @@ -0,0 +1,156 @@ +import argparse +import pathlib +import sys + +import pdocs.doc +import pdocs.extract +import pdocs.render +import pdocs.static +import pdocs.version +import pdocs.web + +parser = argparse.ArgumentParser( + description="Automatically generate API docs for Python modules.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) +aa = parser.add_argument +aa("--version", action="version", version="%(prog)s " + pdocs.__version__) +aa( + "modules", + type=str, + metavar="module", + nargs="+", + help="Python module names. These may be import paths resolvable in " + "the current environment, or file paths to a Python module or " + "package.", +) +aa( + "--filter", + type=str, + default=None, + help="When specified, only identifiers containing the name given " + "will be shown in the output. Search is case sensitive. " + "Has no effect when --http is set.", +) +aa("--html", action="store_true", help="When set, the output will be HTML formatted.") +aa( + "--html-dir", + type=str, + default=".", + help="The directory to output HTML files to. This option is ignored when " + "outputting documentation as plain text.", +) +aa( + "--html-no-source", + action="store_true", + help="When set, source code will not be viewable in the generated HTML. " + "This can speed up the time required to document large modules.", +) +aa( + "--overwrite", + action="store_true", + help="Overwrites any existing HTML files instead of producing an error.", +) +aa( + "--all-submodules", + action="store_true", + help="When set, every submodule will be included, regardless of whether " + "__all__ is set and contains the submodule.", +) +aa( + "--external-links", + action="store_true", + help="When set, identifiers to external modules are turned into links. " + "This is automatically set when using --http.", +) +aa( + "--template-dir", + type=str, + default=None, + help="Specify a directory containing Mako templates. " + "Alternatively, put your templates in $XDG_CONFIG_HOME/pdoc and " + "pdoc will automatically find them.", +) +aa( + "--link-prefix", + type=str, + default="", + help="A prefix to use for every link in the generated documentation. " + "No link prefix results in all links being relative. " + "Has no effect when combined with --http.", +) +aa( + "--http", + action="store_true", + help="When set, pdoc will run as an HTTP server providing documentation " + "of all installed modules. Only modules found in PYTHONPATH will be " + "listed.", +) +aa("--http-host", type=str, default="localhost", help="The host on which to run the HTTP server.") +aa("--http-port", type=int, default=8080, help="The port on which to run the HTTP server.") + + +def _eprint(*args, **kwargs): + kwargs["file"] = sys.stderr + print(*args, **kwargs) + + +def run(): + """ Command-line entry point """ + args = parser.parse_args() + + docfilter = None + if args.filter and len(args.filter.strip()) > 0: + search = args.filter.strip() + + def docfilter(o): + rname = o.refname + if rname.find(search) > -1 or search.find(o.name) > -1: + return True + if isinstance(o, pdocs.doc.Class): + return search in o.doc or search in o.doc_init + return False + + roots = [] + for mod in args.modules: + try: + m = pdocs.extract.extract_module(mod) + except pdocs.extract.ExtractError as e: + _eprint(str(e)) + sys.exit(1) + roots.append(m) + + if args.template_dir is not None: + pdocs.doc.tpl_lookup.directories.insert(0, args.template_dir) + if args.http: + args.html = True + args.external_links = True + args.overwrite = True + args.link_prefix = "/" + + if args.http: + # Run the HTTP server. + httpd = pdocs.web.DocServer((args.http_host, args.http_port), args, roots) + print( + "pdoc server ready at http://%s:%d" % (args.http_host, args.http_port), file=sys.stderr + ) + httpd.serve_forever() + httpd.server_close() + elif args.html: + dst = pathlib.Path(args.html_dir) + if not args.overwrite and pdocs.static.would_overwrite(dst, roots): + _eprint("Rendering would overwrite files, but --overwite is not set") + sys.exit(1) + pdocs.static.html_out(dst, roots) + else: + # Plain text + for m in roots: + output = pdocs.render.text(m) + print(output) + + +def main(): + try: + run() + except KeyboardInterrupt: + pass diff --git a/pdocs/doc.py b/pdocs/doc.py new file mode 100644 index 0000000..f017325 --- /dev/null +++ b/pdocs/doc.py @@ -0,0 +1,808 @@ +import ast +import inspect +import typing + +__pdoc__ = {} + + +def _source(obj): + """ + Returns the source code of the Python object `obj` as a list of + lines. This tries to extract the source from the special + `__wrapped__` attribute if it exists. Otherwise, it falls back + to `inspect.getsourcelines`. + + If neither works, then the empty list is returned. + """ + try: + return inspect.getsourcelines(obj.__wrapped__)[0] + except: + pass + try: + return inspect.getsourcelines(obj)[0] + except: + return [] + + +def _var_docstrings(tree, module, cls=None, init=False): + """ + Extracts variable docstrings given `tree` as the abstract syntax, + `module` as a `pdoc.Module` containing `tree` and an option `cls` + as a `pdoc.Class` corresponding to the tree. In particular, `cls` + should be specified when extracting docstrings from a class or an + `__init__` method. Finally, `init` should be `True` when searching + the AST of an `__init__` method so that `_var_docstrings` will only + accept variables starting with `self.` as instance variables. + + A dictionary mapping variable name to a `pdoc.Variable` object is + returned. + """ + vs = {} + children = list(ast.iter_child_nodes(tree)) + for i, child in enumerate(children): + if isinstance(child, ast.Assign) and len(child.targets) == 1: + if not init and isinstance(child.targets[0], ast.Name): + name = child.targets[0].id + elif ( + isinstance(child.targets[0], ast.Attribute) + and isinstance(child.targets[0].value, ast.Name) + and child.targets[0].value.id == "self" + ): + name = child.targets[0].attr + else: + continue + if not _is_exported(name) and name not in getattr(module, "__all__", []): + continue + + docstring = "" + if ( + i + 1 < len(children) + and isinstance(children[i + 1], ast.Expr) + and isinstance(children[i + 1].value, ast.Str) + ): + docstring = children[i + 1].value.s + + vs[name] = Variable(name, module, docstring, cls=cls) + return vs + + +def _is_exported(ident_name): + """ + Returns `True` if `ident_name` matches the export criteria for an + identifier name. + + This should not be used by clients. Instead, use + `pdoc.Module.is_public`. + """ + return not ident_name.startswith("_") + + +def _is_method(cls: typing.Type, method_name: str) -> bool: + """ + Returns `True` if the given method is a regular method, + i.e. it's neither annotated with @classmethod nor @staticmethod. + """ + func = getattr(cls, method_name, None) + if inspect.ismethod(func): + # If the function is already bound, it's a classmethod. + # Regular methods are not bound before initialization. + return False + for c in inspect.getmro(cls): + if method_name in c.__dict__: + return not isinstance(c.__dict__[method_name], staticmethod) + else: + raise ValueError( + "{method_name} not found in {cls}.".format(method_name=method_name, cls=cls) + ) + + +class Doc(object): + """ + A base class for all documentation objects. + + A documentation object corresponds to *something* in a Python module + that has a docstring associated with it. Typically, this only includes + modules, classes, functions and methods. However, `pdoc` adds support + for extracting docstrings from the abstract syntax tree, which means + that variables (module, class or instance) are supported too. + + A special type of documentation object `pdoc.External` is used to + represent identifiers that are not part of the public interface of + a module. (The name "External" is a bit of a misnomer, since it can + also correspond to unexported members of the module, particularly in + a class's ancestor list.) + """ + + def __init__(self, name, module, docstring): + """ + Initializes a documentation object, where `name` is the public + identifier name, `module` is a `pdoc.Module` object, and + `docstring` is a string containing the docstring for `name`. + """ + self.module = module + """ + The module documentation object that this object was defined + in. + """ + + self.name = name + """ + The identifier name for this object. + """ + + self.docstring = inspect.cleandoc(docstring or "") + """ + The docstring for this object. It has already been cleaned + by `inspect.cleandoc`. + """ + + @property + def source(self): + """ + Returns the source code of the Python object `obj` as a list of + lines. This tries to extract the source from the special + `__wrapped__` attribute if it exists. Otherwise, it falls back + to `inspect.getsourcelines`. + + If neither works, then the empty list is returned. + """ + assert False, "subclass responsibility" + + @property + def refname(self): + """ + Returns an appropriate reference name for this documentation + object. Usually this is its fully qualified path. Every + documentation object must provide this property. + + e.g., The refname for this property is + pdoc.Doc.refname. + """ + assert False, "subclass responsibility" + + def __lt__(self, other): + return self.name < other.name + + def is_empty(self): + """ + Returns true if the docstring for this object is empty. + """ + return len(self.docstring.strip()) == 0 + + +class Module(Doc): + """ + Representation of a module's documentation. + """ + + __pdoc__["Module.module"] = "The Python module object." + __pdoc__[ + "Module.name" + ] = """ + The name of this module with respect to the context in which + it was imported. It is always an absolute import path. + """ + + def __init__(self, name, module, parent): + """ + Creates a `Module` documentation object given the actual + module Python object. + """ + super().__init__(name, module, inspect.getdoc(module)) + self.parent = parent + + self.doc = {} + """A mapping from identifier name to a documentation object.""" + + self.refdoc = {} + """ + The same as `pdoc.Module.doc`, but maps fully qualified + identifier names to documentation objects. + """ + + self.submodules = [] + + vardocs = {} + try: + tree = ast.parse(inspect.getsource(self.module)) + vardocs = _var_docstrings(tree, self, cls=None) + except: + pass + self._declared_variables = vardocs.keys() + + public = self.__public_objs() + for name, obj in public.items(): + # Skip any identifiers that already have doco. + if name in self.doc and not self.doc[name].is_empty(): + continue + + # Functions and some weird builtins?, plus methods, classes, + # modules and module level variables. + if inspect.isfunction(obj) or inspect.isbuiltin(obj): + self.doc[name] = Function(name, self, obj) + elif inspect.ismethod(obj): + self.doc[name] = Function(name, self, obj) + elif inspect.isclass(obj): + self.doc[name] = Class(name, self, obj) + elif name in vardocs: + self.doc[name] = vardocs[name] + else: + # Catch all for variables. + self.doc[name] = Variable(name, self, "", cls=None) + + # Now see if we can grab inheritance relationships between classes. + for docobj in self.doc.values(): + if isinstance(docobj, Class): + docobj._fill_inheritance() + + # Build the reference name dictionary. + for basename, docobj in self.doc.items(): + self.refdoc[docobj.refname] = docobj + if isinstance(docobj, Class): + for v in docobj.class_variables(): + self.refdoc[v.refname] = v + for v in docobj.instance_variables(): + self.refdoc[v.refname] = v + for f in docobj.methods(): + self.refdoc[f.refname] = f + for f in docobj.functions(): + self.refdoc[f.refname] = f + + # Finally look for more docstrings in the __pdoc__ override. + for name, docstring in getattr(self.module, "__pdoc__", {}).items(): + refname = "%s.%s" % (self.refname, name) + if docstring is None: + self.doc.pop(name, None) + self.refdoc.pop(refname, None) + continue + + dobj = self.find_ident(refname) + if isinstance(dobj, External): + continue + dobj.docstring = inspect.cleandoc(docstring) + + @property + def source(self): + return _source(self.module) + + @property + def refname(self): + return self.name + + def mro(self, cls): + """ + Returns a method resolution list of ancestor documentation objects + for `cls`, which must be a documentation object. + + The list will contain objects belonging to `pdoc.Class` or + `pdoc.External`. Objects belonging to the former are exported + classes either in this module or in one of its sub-modules. + """ + return [self.find_class(c) for c in inspect.getmro(cls.cls) if c not in (cls.cls, object)] + + def descendents(self, cls): + """ + Returns a descendent list of documentation objects for `cls`, + which must be a documentation object. + + The list will contain objects belonging to `pdoc.Class` or + `pdoc.External`. Objects belonging to the former are exported + classes either in this module or in one of its sub-modules. + """ + if cls.cls == type or not hasattr(cls.cls, "__subclasses__"): + # Is this right? + return [] + + downs = cls.cls.__subclasses__() + return list(map(lambda c: self.find_class(c), downs)) + + def is_public(self, name): + """ + Returns `True` if and only if an identifier with name `name` is + part of the public interface of this module. While the names + of sub-modules are included, identifiers only exported by + sub-modules are not checked. + + `name` should be a fully qualified name, e.g., + pdoc.Module.is_public. + """ + return name in self.refdoc + + def find_class(self, cls): + """ + Given a Python `cls` object, try to find it in this module + or in any of the exported identifiers of the submodules. + """ + for doc_cls in self.classes(): + if cls is doc_cls.cls: + return doc_cls + for module in self.submodules: + doc_cls = module.find_class(cls) + if not isinstance(doc_cls, External): + return doc_cls + return External("%s.%s" % (cls.__module__, cls.__name__)) + + def find_ident(self, name, _seen=None): + """ + Searches this module and **all** of its sub/super-modules for an + identifier with name `name` in its list of exported + identifiers according to `pdoc`. Note that unexported + sub-modules are searched. + + A bare identifier (without `.` separators) will only be checked + for in this module. + + The documentation object corresponding to the identifier is + returned. If one cannot be found, then an instance of + `External` is returned populated with the given identifier. + """ + _seen = _seen or set() + if self in _seen: + return None + _seen.add(self) + + if name == self.refname: + return self + if name in self.refdoc: + return self.refdoc[name] + for module in self.submodules: + o = module.find_ident(name, _seen=_seen) + if not isinstance(o, (External, type(None))): + return o + # Traverse also up-level super-modules + module = self.parent + while module is not None: + o = module.find_ident(name, _seen=_seen) + if not isinstance(o, (External, type(None))): + return o + module = module.parent + return External(name) + + def variables(self): + """ + Returns all documented module level variables in the module + sorted alphabetically as a list of `pdoc.Variable`. + """ + p = lambda o: isinstance(o, Variable) + return sorted(filter(p, self.doc.values())) + + def classes(self): + """ + Returns all documented module level classes in the module + sorted alphabetically as a list of `pdoc.Class`. + """ + p = lambda o: isinstance(o, Class) + return sorted(filter(p, self.doc.values())) + + def functions(self): + """ + Returns all documented module level functions in the module + sorted alphabetically as a list of `pdoc.Function`. + """ + p = lambda o: isinstance(o, Function) + return sorted(filter(p, self.doc.values())) + + def __is_exported(self, name, module): + """ + Returns `True` if and only if `pdoc` considers `name` to be + a public identifier for this module where `name` was defined + in the Python module `module`. + + If this module has an `__all__` attribute, then `name` is + considered to be exported if and only if it is a member of + this module's `__all__` list. + + If `__all__` is not set, then whether `name` is exported or + not is heuristically determined. Firstly, if `name` starts + with an underscore, it will not be considered exported. + Secondly, if `name` was defined in a module other than this + one, it will not be considered exported. In all other cases, + `name` will be considered exported. + """ + if hasattr(self.module, "__all__"): + return name in self.module.__all__ + if not _is_exported(name): + return False + if module is not None and self.module.__name__ != module.__name__: + return name in self._declared_variables + return True + + def __public_objs(self): + """ + Returns a dictionary mapping a public identifier name to a + Python object. + """ + members = dict(inspect.getmembers(self.module)) + return dict( + [ + (name, obj) + for name, obj in members.items() + if self.__is_exported(name, inspect.getmodule(obj)) + ] + ) + + def allmodules(self): + yield self + for i in self.submodules: + yield from i.allmodules() + + def toroot(self): + n = self + while n: + yield n + n = n.parent + + +class Class(Doc): + """ + Representation of a class's documentation. + """ + + def __init__(self, name, module, class_obj): + """ + Same as `pdoc.Doc.__init__`, except `class_obj` must be a + Python class object. The docstring is gathered automatically. + """ + super().__init__(name, module, inspect.getdoc(class_obj)) + + self.cls = class_obj + """The class Python object.""" + + self.doc = {} + """A mapping from identifier name to a `pdoc.Doc` objects.""" + + self.doc_init = {} + """ + A special version of `pdoc.Class.doc` that contains + documentation for instance variables found in the `__init__` + method. + """ + + public = self.__public_objs() + try: + # First try and find docstrings for class variables. + # Then move on to finding docstrings for instance variables. + # This must be optional, since not all modules have source + # code available. + cls_ast = ast.parse(inspect.getsource(self.cls)).body[0] + self.doc = _var_docstrings(cls_ast, self.module, cls=self) + + for n in cls_ast.body if "__init__" in public else []: + if isinstance(n, ast.FunctionDef) and n.name == "__init__": + self.doc_init = _var_docstrings(n, self.module, cls=self, init=True) + break + except: + pass + + # Convert the public Python objects to documentation objects. + for name, obj in public.items(): + # Skip any identifiers that already have doco. + if name in self.doc and not self.doc[name].is_empty(): + continue + if name in self.doc_init: + # Let instance members override class members. + continue + + if inspect.isfunction(obj) or inspect.ismethod(obj): + self.doc[name] = Function( + name, self.module, obj, cls=self, method=_is_method(self.cls, name) + ) + elif isinstance(obj, property): + docstring = getattr(obj, "__doc__", "") + self.doc_init[name] = Variable(name, self.module, docstring, cls=self) + elif not inspect.isbuiltin(obj) and not inspect.isroutine(obj): + if name in getattr(self.cls, "__slots__", []): + self.doc_init[name] = Variable(name, self.module, "", cls=self) + else: + self.doc[name] = Variable(name, self.module, "", cls=self) + + @property + def source(self): + return _source(self.cls) + + @property + def refname(self): + return "%s.%s" % (self.module.refname, self.cls.__name__) + + def class_variables(self): + """ + Returns all documented class variables in the class, sorted + alphabetically as a list of `pdoc.Variable`. + """ + p = lambda o: isinstance(o, Variable) + return sorted(filter(p, self.doc.values())) + + def instance_variables(self): + """ + Returns all instance variables in the class, sorted + alphabetically as a list of `pdoc.Variable`. Instance variables + are attributes of `self` defined in a class's `__init__` + method. + """ + p = lambda o: isinstance(o, Variable) + return sorted(filter(p, self.doc_init.values())) + + def methods(self): + """ + Returns all documented methods as `pdoc.Function` objects in + the class, sorted alphabetically with `__init__` always coming + first. + + Unfortunately, this also includes class methods. + """ + p = lambda o: (isinstance(o, Function) and o.method) + return sorted(filter(p, self.doc.values())) + + def functions(self): + """ + Returns all documented static functions as `pdoc.Function` + objects in the class, sorted alphabetically. + """ + p = lambda o: (isinstance(o, Function) and not o.method) + return sorted(filter(p, self.doc.values())) + + def _fill_inheritance(self): + """ + Traverses this class's ancestor list and attempts to fill in + missing documentation from its ancestor's documentation. + + The first pass connects variables, methods and functions with + their inherited couterparts. (The templates will decide how to + display docstrings.) The second pass attempts to add instance + variables to this class that were only explicitly declared in + a parent class. This second pass is necessary since instance + variables are only discoverable by traversing the abstract + syntax tree. + """ + mro = [c for c in self.module.mro(self) if c != self and isinstance(c, Class)] + + def search(d, fdoc): + for c in mro: + doc = fdoc(c) + if d.name in doc and isinstance(d, type(doc[d.name])): + return doc[d.name] + return None + + for fdoc in (lambda c: c.doc_init, lambda c: c.doc): + for d in fdoc(self).values(): + dinherit = search(d, fdoc) + if dinherit is not None: + d.inherits = dinherit + + # Since instance variables aren't part of a class's members, + # we need to manually deduce inheritance. Oh lawdy. + for c in mro: + for name in filter(lambda n: n not in self.doc_init, c.doc_init): + d = c.doc_init[name] + self.doc_init[name] = Variable(d.name, d.module, "", cls=self) + self.doc_init[name].inherits = d + + def __public_objs(self): + """ + Returns a dictionary mapping a public identifier name to a + Python object. This counts the `__init__` method as being + public. + """ + _pdoc = getattr(self.module.module, "__pdoc__", {}) + + def forced_out(name): + return _pdoc.get("%s.%s" % (self.name, name), False) is None + + def exported(name): + exported = name == "__init__" or _is_exported(name) + return not forced_out(name) and exported + + idents = dict(inspect.getmembers(self.cls)) + return dict([(n, o) for n, o in idents.items() if exported(n)]) + + +class Function(Doc): + """ + Representation of documentation for a Python function or method. + """ + + def __init__(self, name, module, func_obj, cls=None, method=False): + """ + Same as `pdoc.Doc.__init__`, except `func_obj` must be a + Python function object. The docstring is gathered automatically. + + `cls` should be set when this is a method or a static function + beloing to a class. `cls` should be a `pdoc.Class` object. + + `method` should be `True` when the function is a method. In + all other cases, it should be `False`. + """ + super().__init__(name, module, inspect.getdoc(func_obj)) + + self.func = func_obj + """The Python function object.""" + + self.cls = cls + """ + The `pdoc.Class` documentation object if this is a method. If + not, this is None. + """ + + self.method = method + """ + Whether this function is a method or not. + + In particular, static class methods have this set to False. + """ + + @property + def source(self): + return _source(self.func) + + @property + def refname(self): + if self.cls is None: + return "%s.%s" % (self.module.refname, self.name) + else: + return "%s.%s" % (self.cls.refname, self.name) + + def funcdef(self): + """ + Generates the string of keywords used to define the function, for + example `def` or `async def`. + """ + keywords = [] + + if self._is_async(): + keywords.append("async") + + keywords.append("def") + + return " ".join(keywords) + + def _is_async(self): + """ + Returns whether is function is asynchronous, either as a coroutine or an + async generator. + """ + try: + # Both of these are required because coroutines aren't classified as + # async generators and vice versa. + return inspect.iscoroutinefunction(self.func) or inspect.isasyncgenfunction(self.func) + except AttributeError: + return False + + def spec(self): + """ + Returns a nicely formatted spec of the function's parameter + list as a string. This includes argument lists, keyword + arguments and default values. + """ + return ", ".join(self.params()) + + def params(self): + """ + Returns a list where each element is a nicely formatted + parameter of this function. This includes argument lists, + keyword arguments and default values. + """ + + def fmt_param(el): + if isinstance(el, str): + return el + else: + return "(%s)" % (", ".join(map(fmt_param, el))) + + try: + getspec = getattr(inspect, "getfullargspec", inspect.getargspec) + s = getspec(self.func) + except TypeError: + # I guess this is for C builtin functions? + return ["..."] + + params = [] + for i, param in enumerate(s.args): + if s.defaults is not None and len(s.args) - i <= len(s.defaults): + defind = len(s.defaults) - (len(s.args) - i) + params.append("%s=%s" % (param, repr(s.defaults[defind]))) + else: + params.append(fmt_param(param)) + if s.varargs is not None: + params.append("*%s" % s.varargs) + + kwonlyargs = getattr(s, "kwonlyargs", None) + if kwonlyargs: + if s.varargs is None: + params.append("*") + for param in kwonlyargs: + try: + params.append("%s=%s" % (param, repr(s.kwonlydefaults[param]))) + except KeyError: + params.append(param) + + keywords = getattr(s, "varkw", getattr(s, "keywords", None)) + if keywords is not None: + params.append("**%s" % keywords) + # TODO: The only thing now missing for Python 3 are type annotations + return params + + def __lt__(self, other): + # Push __init__ to the top. + if "__init__" in (self.name, other.name): + return self.name != other.name and self.name == "__init__" + else: + return self.name < other.name + + +class Variable(Doc): + """ + Representation of a variable's documentation. This includes + module, class and instance variables. + """ + + def __init__(self, name, module, docstring, cls=None): + """ + Same as `pdoc.Doc.__init__`, except `cls` should be provided + as a `pdoc.Class` object when this is a class or instance + variable. + """ + super().__init__(name, module, docstring) + + self.cls = cls + """ + The `podc.Class` object if this is a class or instance + variable. If not, this is None. + """ + + @property + def source(self): + return [] + + @property + def refname(self): + if self.cls is None: + return "%s.%s" % (self.module.refname, self.name) + else: + return "%s.%s" % (self.cls.refname, self.name) + + +class External(Doc): + """ + A representation of an external identifier. The textual + representation is the same as an internal identifier, but without + any context. (Usually this makes linking more difficult.) + + External identifiers are also used to represent something that is + not exported but appears somewhere in the public interface (like + the ancestor list of a class). + """ + + __pdoc__[ + "External.docstring" + ] = """ + An empty string. External identifiers do not have + docstrings. + """ + __pdoc__[ + "External.module" + ] = """ + Always `None`. External identifiers have no associated + `pdoc.Module`. + """ + __pdoc__[ + "External.name" + ] = """ + Always equivalent to `pdoc.External.refname` since external + identifiers are always expressed in their fully qualified + form. + """ + + def __init__(self, name): + """ + Initializes an external identifier with `name`, where `name` + should be a fully qualified name. + """ + super().__init__(name, None, "") + + @property + def source(self): + return [] + + @property + def refname(self): + return self.name diff --git a/pdocs/extract.py b/pdocs/extract.py new file mode 100644 index 0000000..e9168d1 --- /dev/null +++ b/pdocs/extract.py @@ -0,0 +1,122 @@ +import importlib +import os +import pkgutil +import typing + +import pdocs.doc + + +class ExtractError(Exception): + pass + + +def split_module_spec(spec: str) -> typing.Tuple[str, str]: + """ + Splits a module specification into a base path (which may be empty), and a module name. + + Raises ExtactError if the spec is invalid. + """ + if not spec: + raise ExtractError("Empty module spec.") + if (os.sep in spec) or (os.altsep and os.altsep in spec): + dirname, fname = os.path.split(spec) + if fname.endswith(".py"): + mname, _ = os.path.splitext(fname) + return dirname, mname + else: + if "." in fname: + raise ExtractError( + "Invalid module name {fname}. Mixing path and module specifications " + "is not supported.".format(fname=fname) + ) + return dirname, fname + else: + return "", spec + + +def load_module(basedir: str, module: str) -> typing.Tuple[typing.Any, bool]: + """ + Returns a module object, and whether the module is a package or not. + """ + ispackage = False + if basedir: + mods = module.split(".") + dirname = os.path.join(basedir, *mods[:-1]) + modname = mods[-1] + + pkgloc = os.path.join(dirname, modname, "__init__.py") + fileloc = os.path.join(dirname, modname + ".py") + + if os.path.exists(pkgloc): + location, ispackage = pkgloc, True + elif os.path.exists(fileloc): + location, ispackage = fileloc, False + else: + raise ExtractError( + "Module {module} not found in {basedir}".format(module=module, basedir=basedir) + ) + + ispec = importlib.util.spec_from_file_location(modname, location) + mobj = importlib.util.module_from_spec(ispec) + try: + # This can literally raise anything + ispec.loader.exec_module(mobj) # type: ignore + except Exception as e: + raise ExtractError("Error importing {location}: {e}".format(location=location, e=e)) + return mobj, ispackage + else: + try: + # This can literally raise anything + m = importlib.import_module(module) + except ImportError: + raise ExtractError("Module not found: {module}".format(module=module)) + except Exception as e: + raise ExtractError("Error importing {module}: {e}".format(module=module, e=e)) + # This is the only case where we actually have to test whether we're a package + if getattr(m, "__package__", False) and getattr(m, "__path__", False): + ispackage = True + return m, ispackage + + +def submodules(dname: str, mname: str) -> typing.Sequence[str]: + """ + Returns a list of fully qualified submodules within a package, given a + base directory and a fully qualified module name. + """ + loc = os.path.join(dname, *mname.split(".")) + ret = [] + for mi in pkgutil.iter_modules([loc], prefix=mname + "."): + if isinstance(mi, tuple): + # Python 3.5 compat + ret.append(mi[1]) + else: + ret.append(mi.name) + ret.sort() + return ret + + +def _extract_module(dname: str, mname: str, parent=None) -> typing.Any: + m, pkg = load_module(dname, mname) + mod = pdocs.doc.Module(mname, m, parent) + if pkg: + for i in submodules(dname, mname): + mod.submodules.append(_extract_module(dname, i, parent=mod)) + return mod + + +def extract_module(spec: str): + """ + Extracts and returns a module object. The spec argument can have the + following forms: + + Simple module: "foo.bar" + Module path: "./path/to/module" + File path: "./path/to/file.py" + + This function always invalidates caches to enable hot load and reload. + + May raise ExtactError. + """ + importlib.invalidate_caches() + dname, mname = split_module_spec(spec) + return _extract_module(dname, mname) diff --git a/pdocs/html_helpers.py b/pdocs/html_helpers.py new file mode 100644 index 0000000..608291a --- /dev/null +++ b/pdocs/html_helpers.py @@ -0,0 +1,162 @@ +import os +import re +import sys + +import markdown +import pygments +import pygments.formatters +import pygments.lexers + +import pdocs.doc +import pdocs.render + +# From language reference, but adds '.' to allow fully qualified names. +pyident = re.compile("^[a-zA-Z_][a-zA-Z0-9_.]+$") +indent = re.compile("^\s*") + + +def decode(s): + if sys.version_info[0] < 3 and isinstance(s, str): + return s.decode("utf-8", "ignore") + return s + + +def ident(s): + return '%s' % s + + +def sourceid(dobj): + return "source-%s" % dobj.refname + + +def clean_source_lines(lines): + """ + Cleans the source code so that pygments can render it well. + + Returns one string with all of the source code. + """ + base_indent = len(indent.match(lines[0]).group(0)) + base_indent = 0 + for line in lines: + if len(line.strip()) > 0: + base_indent = len(indent.match(lines[0]).group(0)) + break + lines = [line[base_indent:] for line in lines] + + if sys.version_info[0] < 3: + pylex = pygments.lexers.PythonLexer() + else: + pylex = pygments.lexers.Python3Lexer() + + htmlform = pygments.formatters.HtmlFormatter(cssclass="codehilite") + return pygments.highlight("".join(lines), pylex, htmlform) + + +def linkify(parent, match, link_prefix): + matched = match.group(0) + ident = matched[1:-1] + name, url = lookup(parent, ident, link_prefix) + if name is None: + return matched + return "[`%s`](%s)" % (name, url) + + +def mark(s, module_list=None, linky=True): + if linky: + s, _ = re.subn("\b\n\b", " ", s) + # if not module_list: + # s, _ = re.subn("`[^`]+`", linkify, s) + + extensions = ["fenced-code-blocks"] + s = markdown2.markdown(s.strip(), extras=extensions) + return s + + +def glimpse(s, length=100): + if len(s) < length: + return s + return s[0:length] + "..." + + +def module_url(parent, m, link_prefix): + """ + Returns a URL for `m`, which must be an instance of `Module`. + Also, `m` must be a submodule of the module being documented. + + Namely, '.' import separators are replaced with '/' URL + separators. Also, packages are translated as directories + containing `index.html` corresponding to the `__init__` module, + while modules are translated as regular HTML files with an + `.m.html` suffix. (Given default values of + `pdoc.html_module_suffix` and `pdoc.html_package_name`.) + """ + if parent.name == m.name: + return "" + + base = m.name.replace(".", "/") + if len(link_prefix) == 0: + base = os.path.relpath(base, parent.name.replace(".", "/")) + url = base[len("../") :] if base.startswith("../") else "" if base == ".." else base + if m.submodules: + index = pdocs.render.html_package_name + url = url + "/" + index if url else index + else: + url += pdocs.render.html_module_suffix + return link_prefix + url + + +def external_url(refname): + """ + Attempts to guess an absolute URL for the external identifier + given. + + Note that this just returns the refname with an ".ext" suffix. + It will be up to whatever is interpreting the URLs to map it + to an appropriate documentation page. + """ + return "/%s.ext" % refname + + +def is_external_linkable(name): + return pyident.match(name) and "." in name + + +def lookup(module, refname, link_prefix): + """ + Given a fully qualified identifier name, return its refname + with respect to the current module and a value for a `href` + attribute. If `refname` is not in the public interface of + this module or its submodules, then `None` is returned for + both return values. (Unless this module has enabled external + linking.) + + In particular, this takes into account sub-modules and external + identifiers. If `refname` is in the public API of the current + module, then a local anchor link is given. If `refname` is in the + public API of a sub-module, then a link to a different page with + the appropriate anchor is given. Otherwise, `refname` is + considered external and no link is used. + """ + d = module.find_ident(refname) + if isinstance(d, pdocs.doc.External): + if is_external_linkable(refname): + return d.refname, external_url(d.refname) + else: + return None, None + if isinstance(d, pdocs.doc.Module): + return d.refname, module_url(module, d, link_prefix) + if module.is_public(d.refname): + return d.name, "#%s" % d.refname + return d.refname, "%s#%s" % (module_url(module, d.module, link_prefix), d.refname) + + +def link(parent, refname, link_prefix): + """ + A convenience wrapper around `href` to produce the full + `a` tag if `refname` is found. Otherwise, plain text of + `refname` is returned. + """ + name, url = lookup(parent, refname, link_prefix) + if name is None: + return refname + return '%s' % (url, name) diff --git a/pdocs/render.py b/pdocs/render.py new file mode 100644 index 0000000..ca3c43e --- /dev/null +++ b/pdocs/render.py @@ -0,0 +1,114 @@ +import os.path +import re +import typing + +from mako.exceptions import TopLevelLookupException +from mako.lookup import TemplateLookup + +import pdocs.doc + +html_module_suffix = ".m.html" +""" +The suffix to use for module HTML files. By default, this is set to +`.m.html`, where the extra `.m` is used to differentiate a package's +`index.html` from a submodule called `index`. +""" + +html_package_name = "index.html" +""" +The file name to use for a package's `__init__.py` module. +""" + +_template_path = [os.path.join(os.path.dirname(__file__), "templates")] +""" +A list of paths to search for Mako templates used to produce the +plain text and HTML output. Each path is tried until a template is +found. +""" + +tpl_lookup = TemplateLookup( + directories=_template_path, cache_args={"cached": True, "cache_type": "memory"} +) +""" +A `mako.lookup.TemplateLookup` object that knows how to load templates +from the file system. You may add additional paths by modifying the +object's `directories` attribute. +""" + + +def _get_tpl(name): + """ + Returns the Mako template with the given name. If the template cannot be + found, a nicer error message is displayed. + """ + try: + t = tpl_lookup.get_template(name) + except TopLevelLookupException: + locs = [os.path.join(p, name.lstrip("/")) for p in _template_path] + raise IOError(2, "No template at any of: %s" % ", ".join(locs)) + return t + + +def html_index(roots: typing.Sequence[pdocs.doc.Module], link_prefix: str = "/") -> str: + """ + Render an HTML module index. + """ + t = _get_tpl("/html_index.mako") + t = t.render(roots=roots, link_prefix=link_prefix) + return t.strip() + + +def html_module( + mod: pdocs.doc.Module, external_links: bool = False, link_prefix: str = "/", source: bool = True +) -> str: + """ + Returns the documentation for the module `module_name` in HTML + format. The module must be importable. + + `docfilter` is an optional predicate that controls which + documentation objects are shown in the output. It is a single + argument function that takes a documentation object and returns + `True` or `False`. If `False`, that object will not be included in + the output. + + If `allsubmodules` is `True`, then every submodule of this module + that can be found will be included in the documentation, regardless + of whether `__all__` contains it. + + If `external_links` is `True`, then identifiers to external modules + are always turned into links. + + If `link_prefix` is `True`, then all links will have that prefix. + Otherwise, links are always relative. + + If `source` is `True`, then source code will be retrieved for + every Python object whenever possible. This can dramatically + decrease performance when documenting large modules. + """ + t = _get_tpl("/html_module.mako") + t = t.render( + module=mod, external_links=external_links, link_prefix=link_prefix, show_source_code=source + ) + return t.strip() + + +def text( + mod: pdocs.doc.Module, docfilter: typing.Optional[str] = None, allsubmodules: bool = False +) -> str: + """ + Returns the documentation for the module `module_name` in plain + text format. The module must be importable. + + `docfilter` is an optional predicate that controls which + documentation objects are shown in the output. It is a single + argument function that takes a documentation object and returns + True of False. If False, that object will not be included in the + output. + + If `allsubmodules` is `True`, then every submodule of this module + that can be found will be included in the documentation, regardless + of whether `__all__` contains it. + """ + t = _get_tpl("/text.mako") + text, _ = re.subn("\n\n\n+", "\n\n", t.render(module=mod).strip()) + return text diff --git a/pdocs/static.py b/pdocs/static.py new file mode 100644 index 0000000..7538a5e --- /dev/null +++ b/pdocs/static.py @@ -0,0 +1,81 @@ +import pathlib +import typing + +import pdocs.doc +import pdocs.render + + +class StaticError(Exception): + pass + + +def module_to_path(m: pdocs.doc.Module) -> pathlib.Path: + """ + Calculates the filesystem path for the static output of a given module. + """ + p = pathlib.Path(*m.name.split(".")) + if m.submodules: + p /= "index.html" + else: + if p.stem == "index": + p = p.with_suffix(".m.html") + else: + p = p.with_suffix(".html") + return p + + +def path_to_module( + roots: typing.Sequence[pdocs.doc.Module], path: pathlib.Path +) -> pdocs.doc.Module: + """ + Retrieves the matching module for a given path from a module tree. + """ + if path.suffix == ".html": + path = path.with_suffix("") + parts = list(path.parts) + if parts[-1] == "index": + parts = parts[:-1] + elif parts[-1] == "index.m": + parts[-1] = "index" + for root in roots: + mod = root.find_ident(".".join(parts)) + if isinstance(mod, pdocs.doc.Module): + return mod + raise StaticError("No matching module for {path}".format(path=path)) + + +def would_overwrite(dst: pathlib.Path, roots: typing.Sequence[pdocs.doc.Module]) -> bool: + """ + Would rendering root to dst overwrite any file? + """ + if len(roots) > 1: + p = dst / "index.html" + if p.exists(): + return True + for root in roots: + for m in root.allmodules(): + p = dst.joinpath(module_to_path(m)) + if p.exists(): + return True + return False + + +def html_out( + dst: pathlib.Path, + roots: typing.Sequence[pdocs.doc.Module], + external_links: bool = True, + link_prefix: str = "", + source: bool = False, +): + if len(roots) > 1: + p = dst / "index.html" + idx = pdocs.render.html_index(roots, link_prefix=link_prefix) + p.write_text(idx, encoding="utf-8") + for root in roots: + for m in root.allmodules(): + p = dst.joinpath(module_to_path(m)) + p.parent.mkdir(parents=True, exist_ok=True) + out = pdocs.render.html_module( + m, external_links=external_links, link_prefix=link_prefix, source=source + ) + p.write_text(out, encoding="utf-8") diff --git a/pdocs/templates/LICENSE b/pdocs/templates/LICENSE new file mode 100644 index 0000000..294e91d --- /dev/null +++ b/pdocs/templates/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) HTML5 Boilerplate + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pdocs/templates/README.md b/pdocs/templates/README.md new file mode 100644 index 0000000..a396ff5 --- /dev/null +++ b/pdocs/templates/README.md @@ -0,0 +1,3 @@ +The license included in this directory is for +[HTML5 Boiler Plate](http://html5boilerplate.com). Some of the HTML and CSS +used here is derived from that project. diff --git a/pdocs/templates/css.mako b/pdocs/templates/css.mako new file mode 100644 index 0000000..1c3ac6c --- /dev/null +++ b/pdocs/templates/css.mako @@ -0,0 +1,928 @@ +<%def name="pdoc()"> + html, body { + margin: 0; + padding: 0; + min-height: 100%; + } + body { + background: #fff; + font-family: "Source Sans Pro", "Helvetica Neueue", Helvetica, sans; + font-weight: 300; + font-size: 16px; + line-height: 1.6em; + } + #content { + width: 70%; + max-width: 850px; + float: left; + padding: 30px 60px; + border-left: 1px solid #ddd; + } + #sidebar { + width: 25%; + float: left; + padding: 30px; + overflow: hidden; + } + #nav { + font-size: 130%; + margin: 0 0 15px 0; + } + + #top { + display: block; + position: fixed; + bottom: 5px; + left: 5px; + font-size: .85em; + text-transform: uppercase; + } + + #footer { + font-size: .75em; + padding: 5px 30px; + border-top: 1px solid #ddd; + text-align: right; + } + #footer p { + margin: 0 0 0 30px; + display: inline-block; + } + + h1, h2, h3, h4, h5 { + font-weight: 300; + } + h1 { + font-size: 2.5em; + line-height: 1.1em; + margin: 0 0 .50em 0; + } + + h2 { + font-size: 1.75em; + margin: 1em 0 .50em 0; + } + + h3 { + margin: 25px 0 10px 0; + } + + h4 { + margin: 0; + font-size: 105%; + } + + a { + color: #058; + text-decoration: none; + transition: color .3s ease-in-out; + } + + a:hover { + color: #e08524; + transition: color .3s ease-in-out; + } + + pre, code, .mono, .name { + font-family: "Ubuntu Mono", "Cousine", "DejaVu Sans Mono", monospace; + } + + .title .name { + font-weight: bold; + } + .section-title { + margin-top: 2em; + } + .ident { + color: #900; + } + + code { + background: #f9f9f9; + } + + pre { + background: #fefefe; + border: 1px solid #ddd; + box-shadow: 2px 2px 0 #f3f3f3; + margin: 0 30px; + padding: 15px 30px; + } + + .codehilite { + margin: 0 30px 10px 30px; + } + + .codehilite pre { + margin: 0; + } + .codehilite .err { background: #ff3300; color: #fff !important; } + + table#module-list { + font-size: 110%; + } + + table#module-list tr td:first-child { + padding-right: 10px; + white-space: nowrap; + } + + table#module-list td { + vertical-align: top; + padding-bottom: 8px; + } + + table#module-list td p { + margin: 0 0 7px 0; + } + + .def { + display: table; + } + + .def p { + display: table-cell; + vertical-align: top; + text-align: left; + } + + .def p:first-child { + white-space: nowrap; + } + + .def p:last-child { + width: 100%; + } + + + #index { + list-style-type: none; + margin: 0; + padding: 0; + } + ul#index .class_name { + /* font-size: 110%; */ + font-weight: bold; + } + #index ul { + margin: 0; + } + + .item { + margin: 0 0 15px 0; + } + + .item .class { + margin: 0 0 25px 30px; + } + + .item .class ul.class_list { + margin: 0 0 20px 0; + } + + .item .name { + background: #fafafa; + margin: 0; + font-weight: bold; + padding: 5px 10px; + border-radius: 3px; + display: inline-block; + min-width: 40%; + } + .item .name:hover { + background: #f6f6f6; + } + + .item .empty_desc { + margin: 0 0 5px 0; + padding: 0; + } + + .item .inheritance { + margin: 3px 0 0 30px; + } + + .item .inherited { + color: #666; + } + + .item .desc { + padding: 0 8px; + margin: 0; + } + + .item .desc p { + margin: 0 0 10px 0; + } + + .source_cont { + margin: 0; + padding: 0; + } + + .source_link a { + background: #ffc300; + font-weight: 400; + font-size: .75em; + text-transform: uppercase; + color: #fff; + text-shadow: 1px 1px 0 #f4b700; + + padding: 3px 8px; + border-radius: 2px; + transition: background .3s ease-in-out; + } + .source_link a:hover { + background: #FF7200; + text-shadow: none; + transition: background .3s ease-in-out; + } + + .source { + display: none; + max-height: 600px; + overflow-y: scroll; + margin-bottom: 15px; + } + + .source .codehilite { + margin: 0; + } + + .desc h1, .desc h2, .desc h3 { + font-size: 100% !important; + } + .clear { + clear: both; + } + + @media all and (max-width: 950px) { + #sidebar { + width: 35%; + } + #content { + width: 65%; + } + } + @media all and (max-width: 650px) { + #top { + display: none; + } + #sidebar { + float: none; + width: auto; + } + #content { + float: none; + width: auto; + padding: 30px; + } + + #index ul { + padding: 0; + margin-bottom: 15px; + } + #index ul li { + display: inline-block; + margin-right: 30px; + } + #footer { + text-align: left; + } + #footer p { + display: block; + margin: inherit; + } + } + + /*****************************/ + +<%def name="pre()"> +* { + box-sizing: border-box; +} +/*! normalize.css v1.1.1 | MIT License | git.io/normalize */ + +/* ========================================================================== + HTML5 display definitions + ========================================================================== */ + +/** + * Correct `block` display not defined in IE 6/7/8/9 and Firefox 3. + */ + +article, +aside, +details, +figcaption, +figure, +footer, +header, +hgroup, +main, +nav, +section, +summary { + display: block; +} + +/** + * Correct `inline-block` display not defined in IE 6/7/8/9 and Firefox 3. + */ + +audio, +canvas, +video { + display: inline-block; + *display: inline; + *zoom: 1; +} + +/** + * Prevent modern browsers from displaying `audio` without controls. + * Remove excess height in iOS 5 devices. + */ + +audio:not([controls]) { + display: none; + height: 0; +} + +/** + * Address styling not present in IE 7/8/9, Firefox 3, and Safari 4. + * Known issue: no IE 6 support. + */ + +[hidden] { + display: none; +} + +/* ========================================================================== + Base + ========================================================================== */ + +/** + * 1. Prevent system color scheme's background color being used in Firefox, IE, + * and Opera. + * 2. Prevent system color scheme's text color being used in Firefox, IE, and + * Opera. + * 3. Correct text resizing oddly in IE 6/7 when body `font-size` is set using + * `em` units. + * 4. Prevent iOS text size adjust after orientation change, without disabling + * user zoom. + */ + +html { + background: #fff; /* 1 */ + color: #000; /* 2 */ + font-size: 100%; /* 3 */ + -webkit-text-size-adjust: 100%; /* 4 */ + -ms-text-size-adjust: 100%; /* 4 */ +} + +/** + * Address `font-family` inconsistency between `textarea` and other form + * elements. + */ + +html, +button, +input, +select, +textarea { + font-family: sans-serif; +} + +/** + * Address margins handled incorrectly in IE 6/7. + */ + +body { + margin: 0; +} + +/* ========================================================================== + Links + ========================================================================== */ + +/** + * Address `outline` inconsistency between Chrome and other browsers. + */ + +a:focus { + outline: thin dotted; +} + +/** + * Improve readability when focused and also mouse hovered in all browsers. + */ + +a:active, +a:hover { + outline: 0; +} + +/* ========================================================================== + Typography + ========================================================================== */ + +/** + * Address font sizes and margins set differently in IE 6/7. + * Address font sizes within `section` and `article` in Firefox 4+, Safari 5, + * and Chrome. + */ + +h1 { + font-size: 2em; + margin: 0.67em 0; +} + +h2 { + font-size: 1.5em; + margin: 0.83em 0; +} + +h3 { + font-size: 1.17em; + margin: 1em 0; +} + +h4 { + font-size: 1em; + margin: 1.33em 0; +} + +h5 { + font-size: 0.83em; + margin: 1.67em 0; +} + +h6 { + font-size: 0.67em; + margin: 2.33em 0; +} + +/** + * Address styling not present in IE 7/8/9, Safari 5, and Chrome. + */ + +abbr[title] { + border-bottom: 1px dotted; +} + +/** + * Address style set to `bolder` in Firefox 3+, Safari 4/5, and Chrome. + */ + +b, +strong { + font-weight: bold; +} + +blockquote { + margin: 1em 40px; +} + +/** + * Address styling not present in Safari 5 and Chrome. + */ + +dfn { + font-style: italic; +} + +/** + * Address differences between Firefox and other browsers. + * Known issue: no IE 6/7 normalization. + */ + +hr { + -moz-box-sizing: content-box; + box-sizing: content-box; + height: 0; +} + +/** + * Address styling not present in IE 6/7/8/9. + */ + +mark { + background: #ff0; + color: #000; +} + +/** + * Address margins set differently in IE 6/7. + */ + +p, +pre { + margin: 1em 0; +} + +/** + * Correct font family set oddly in IE 6, Safari 4/5, and Chrome. + */ + +code, +kbd, +pre, +samp { + font-family: monospace, serif; + _font-family: 'courier new', monospace; + font-size: 1em; +} + +/** + * Improve readability of pre-formatted text in all browsers. + */ + +pre { + white-space: pre; + white-space: pre-wrap; + word-wrap: break-word; +} + +/** + * Address CSS quotes not supported in IE 6/7. + */ + +q { + quotes: none; +} + +/** + * Address `quotes` property not supported in Safari 4. + */ + +q:before, +q:after { + content: ''; + content: none; +} + +/** + * Address inconsistent and variable font size in all browsers. + */ + +small { + font-size: 80%; +} + +/** + * Prevent `sub` and `sup` affecting `line-height` in all browsers. + */ + +sub, +sup { + font-size: 75%; + line-height: 0; + position: relative; + vertical-align: baseline; +} + +sup { + top: -0.5em; +} + +sub { + bottom: -0.25em; +} + +/* ========================================================================== + Lists + ========================================================================== */ + +/** + * Address margins set differently in IE 6/7. + */ + +dl, +menu, +ol, +ul { + margin: 1em 0; +} + +dd { + margin: 0 0 0 40px; +} + +/** + * Address paddings set differently in IE 6/7. + */ + +menu, +ol, +ul { + padding: 0 0 0 40px; +} + +/** + * Correct list images handled incorrectly in IE 7. + */ + +nav ul, +nav ol { + list-style: none; + list-style-image: none; +} + +/* ========================================================================== + Embedded content + ========================================================================== */ + +/** + * 1. Remove border when inside `a` element in IE 6/7/8/9 and Firefox 3. + * 2. Improve image quality when scaled in IE 7. + */ + +img { + border: 0; /* 1 */ + -ms-interpolation-mode: bicubic; /* 2 */ +} + +/** + * Correct overflow displayed oddly in IE 9. + */ + +svg:not(:root) { + overflow: hidden; +} + +/* ========================================================================== + Figures + ========================================================================== */ + +/** + * Address margin not present in IE 6/7/8/9, Safari 5, and Opera 11. + */ + +figure { + margin: 0; +} + +/* ========================================================================== + Forms + ========================================================================== */ + +/** + * Correct margin displayed oddly in IE 6/7. + */ + +form { + margin: 0; +} + +/** + * Define consistent border, margin, and padding. + */ + +fieldset { + border: 1px solid #c0c0c0; + margin: 0 2px; + padding: 0.35em 0.625em 0.75em; +} + +/** + * 1. Correct color not being inherited in IE 6/7/8/9. + * 2. Correct text not wrapping in Firefox 3. + * 3. Correct alignment displayed oddly in IE 6/7. + */ + +legend { + border: 0; /* 1 */ + padding: 0; + white-space: normal; /* 2 */ + *margin-left: -7px; /* 3 */ +} + +/** + * 1. Correct font size not being inherited in all browsers. + * 2. Address margins set differently in IE 6/7, Firefox 3+, Safari 5, + * and Chrome. + * 3. Improve appearance and consistency in all browsers. + */ + +button, +input, +select, +textarea { + font-size: 100%; /* 1 */ + margin: 0; /* 2 */ + vertical-align: baseline; /* 3 */ + *vertical-align: middle; /* 3 */ +} + +/** + * Address Firefox 3+ setting `line-height` on `input` using `!important` in + * the UA stylesheet. + */ + +button, +input { + line-height: normal; +} + +/** + * Address inconsistent `text-transform` inheritance for `button` and `select`. + * All other form control elements do not inherit `text-transform` values. + * Correct `button` style inheritance in Chrome, Safari 5+, and IE 6+. + * Correct `select` style inheritance in Firefox 4+ and Opera. + */ + +button, +select { + text-transform: none; +} + +/** + * 1. Avoid the WebKit bug in Android 4.0.* where (2) destroys native `audio` + * and `video` controls. + * 2. Correct inability to style clickable `input` types in iOS. + * 3. Improve usability and consistency of cursor style between image-type + * `input` and others. + * 4. Remove inner spacing in IE 7 without affecting normal text inputs. + * Known issue: inner spacing remains in IE 6. + */ + +button, +html input[type="button"], /* 1 */ +input[type="reset"], +input[type="submit"] { + -webkit-appearance: button; /* 2 */ + cursor: pointer; /* 3 */ + *overflow: visible; /* 4 */ +} + +/** + * Re-set default cursor for disabled elements. + */ + +button[disabled], +html input[disabled] { + cursor: default; +} + +/** + * 1. Address box sizing set to content-box in IE 8/9. + * 2. Remove excess padding in IE 8/9. + * 3. Remove excess padding in IE 7. + * Known issue: excess padding remains in IE 6. + */ + +input[type="checkbox"], +input[type="radio"] { + box-sizing: border-box; /* 1 */ + padding: 0; /* 2 */ + *height: 13px; /* 3 */ + *width: 13px; /* 3 */ +} + +/** + * 1. Address `appearance` set to `searchfield` in Safari 5 and Chrome. + * 2. Address `box-sizing` set to `border-box` in Safari 5 and Chrome + * (include `-moz` to future-proof). + */ + +input[type="search"] { + -webkit-appearance: textfield; /* 1 */ + -moz-box-sizing: content-box; + -webkit-box-sizing: content-box; /* 2 */ + box-sizing: content-box; +} + +/** + * Remove inner padding and search cancel button in Safari 5 and Chrome + * on OS X. + */ + +input[type="search"]::-webkit-search-cancel-button, +input[type="search"]::-webkit-search-decoration { + -webkit-appearance: none; +} + +/** + * Remove inner padding and border in Firefox 3+. + */ + +button::-moz-focus-inner, +input::-moz-focus-inner { + border: 0; + padding: 0; +} + +/** + * 1. Remove default vertical scrollbar in IE 6/7/8/9. + * 2. Improve readability and alignment in all browsers. + */ + +textarea { + overflow: auto; /* 1 */ + vertical-align: top; /* 2 */ +} + +/* ========================================================================== + Tables + ========================================================================== */ + +/** + * Remove most spacing between table cells. + */ + +table { + border-collapse: collapse; + border-spacing: 0; +} + + +<%def name="post()"> +/* ========================================================================== + EXAMPLE Media Queries for Responsive Design. + These examples override the primary ('mobile first') styles. + Modify as content requires. + ========================================================================== */ + +@media only screen and (min-width: 35em) { + /* Style adjustments for viewports that meet the condition */ +} + +@media print, + (-o-min-device-pixel-ratio: 5/4), + (-webkit-min-device-pixel-ratio: 1.25), + (min-resolution: 120dpi) { + /* Style adjustments for high resolution devices */ +} + +/* ========================================================================== + Print styles. + Inlined to avoid required HTTP connection: h5bp.com/r + ========================================================================== */ + +@media print { + * { + background: transparent !important; + color: #000 !important; /* Black prints faster: h5bp.com/s */ + box-shadow: none !important; + text-shadow: none !important; + } + + a, + a:visited { + text-decoration: underline; + } + + a[href]:after { + content: " (" attr(href) ")"; + } + + abbr[title]:after { + content: " (" attr(title) ")"; + } + + /* + * Don't show links for images, or javascript/internal links + */ + + .ir a:after, + a[href^="javascript:"]:after, + a[href^="#"]:after { + content: ""; + } + + pre, + blockquote { + border: 1px solid #999; + page-break-inside: avoid; + } + + thead { + display: table-header-group; /* h5bp.com/t */ + } + + tr, + img { + page-break-inside: avoid; + } + + img { + max-width: 100% !important; + } + + @page { + margin: 0.5cm; + } + + p, + h2, + h3 { + orphans: 3; + widows: 3; + } + + h2, + h3 { + page-break-after: avoid; + } +} + diff --git a/pdocs/templates/html_frame.mako b/pdocs/templates/html_frame.mako new file mode 100644 index 0000000..b3a0c4b --- /dev/null +++ b/pdocs/templates/html_frame.mako @@ -0,0 +1,52 @@ +## -*- coding: utf-8 -*- +<%! +import pygments +import pdoc +import pdoc.html_helpers as hh +%> + + + + + + + + <%block name="title"/> + + + <%namespace name="css" file="css.mako" /> + + + + + + + + +Top +
+ ${next.body()} +
+ +
+ + diff --git a/pdocs/templates/html_index.mako b/pdocs/templates/html_index.mako new file mode 100644 index 0000000..090037f --- /dev/null +++ b/pdocs/templates/html_index.mako @@ -0,0 +1,31 @@ +## -*- coding: utf-8 -*- +<%! +import pygments +import pdoc.doc +import pdoc.html_helpers as hh +%> + +<%inherit file="html_frame.mako"/> + +<%def name="show_module_list(roots)"> +

Python module list

+ + % for root in roots: + + + + + % endfor +
${root.name} + % if len(root.docstring.strip()) > 0: +
${root.docstring | hh.mark}
+ % endif +
+ + +<%block name="title"> + Python module index + + + +
${show_module_list(roots)}
\ No newline at end of file diff --git a/pdocs/templates/html_module.mako b/pdocs/templates/html_module.mako new file mode 100644 index 0000000..2b6b8c5 --- /dev/null +++ b/pdocs/templates/html_module.mako @@ -0,0 +1,253 @@ +## -*- coding: utf-8 -*- +<%! +import pygments +import pdoc.doc +import pdoc.html_helpers as hh +%> + +<%inherit file="html_frame.mako"/> + +<%def name="show_source(d)"> + % if show_source_code and d.source is not None and len(d.source) > 0: + +
+ ${hh.decode(hh.clean_source_lines(d.source))} +
+ % endif + + +<%def name="show_desc(d, limit=None)"> + <% + inherits = (hasattr(d, 'inherits') + and (len(d.docstring) == 0 + or d.docstring == d.inherits.docstring)) + docstring = (d.inherits.docstring if inherits else d.docstring).strip() + if limit is not None: + docstring = hh.glimpse(docstring, limit) + %> + % if len(docstring) > 0: + % if inherits: +
${docstring | hh.mark}
+ % else: +
${docstring | hh.mark}
+ % endif + % endif + % if not isinstance(d, pdoc.doc.Module): +
${show_source(d)}
+ % endif + + +<%def name="show_inheritance(d)"> + % if hasattr(d, 'inherits'): +

+ Inheritance: + % if hasattr(d.inherits, 'cls'): + ${hh.link(module, d.inherits.cls.refname, link_prefix)}.${hh.link(module, d.inherits.refname, link_prefix)} + % else: + ${hh.link(module, d.inherits.refname), link_prefix} + % endif +

+ % endif + + +<%def name="show_column_list(items, numcols=3)"> + + + +<%def name="show_module(module)"> + <% + variables = module.variables() + classes = module.classes() + functions = module.functions() + submodules = module.submodules + %> + + <%def name="show_func(f)"> +
+
+

${f.funcdef()} ${hh.ident(f.name)}(

${f.spec() | h})

+
+ ${show_inheritance(f)} + ${show_desc(f)} +
+ + + % if 'http_server' in context.keys() and http_server: + + % endif + +
+

${module.name} module

+ ${module.docstring | hh.mark} + ${show_source(module)} +
+ +
+ % if len(variables) > 0: +

Module variables

+ % for v in variables: +
+

var ${hh.ident(v.name)}

+ ${show_desc(v)} +
+ % endfor + % endif + + % if len(functions) > 0: +

Functions

+ % for f in functions: + ${show_func(f)} + % endfor + % endif + + % if len(classes) > 0: +

Classes

+ % for c in classes: + <% + class_vars = c.class_variables() + smethods = c.functions() + inst_vars = c.instance_variables() + methods = c.methods() + mro = c.module.mro(c) + %> +
+

class ${hh.ident(c.name)}

+ ${show_desc(c)} + +
+ % if len(mro) > 0: +

Ancestors (in MRO)

+
    + % for cls in mro: +
  • ${hh.link(module, cls.refname, link_prefix)}
  • + % endfor +
+ % endif + % if len(class_vars) > 0: +

Class variables

+ % for v in class_vars: +
+

var ${hh.ident(v.name)}

+ ${show_inheritance(v)} + ${show_desc(v)} +
+ % endfor + % endif + % if len(smethods) > 0: +

Static methods

+ % for f in smethods: + ${show_func(f)} + % endfor + % endif + % if len(inst_vars) > 0: +

Instance variables

+ % for v in inst_vars: +
+

var ${hh.ident(v.name)}

+ ${show_inheritance(v)} + ${show_desc(v)} +
+ % endfor + % endif + % if len(methods) > 0: +

Methods

+ % for f in methods: + ${show_func(f)} + % endfor + % endif +
+
+ % endfor + % endif + + % if len(submodules) > 0: +

Sub-modules

+ % for m in submodules: +
+

${hh.link(module, m.refname, link_prefix)}

+ ${show_desc(m, limit=300)} +
+ % endfor + % endif +
+ + +<%def name="module_index(module)"> + <% + variables = module.variables() + classes = module.classes() + functions = module.functions() + submodules = module.submodules + parent = module.parent + %> + + + +<%block name="title"> + ${module.name} API documentation + + + +${module_index(module)} +
${show_module(module)}
\ No newline at end of file diff --git a/pdocs/templates/text.mako b/pdocs/templates/text.mako new file mode 100644 index 0000000..b3ac142 --- /dev/null +++ b/pdocs/templates/text.mako @@ -0,0 +1,145 @@ +## Define mini-templates for each portion of the doco. + +<%! + import re + + def indent(s, spaces=4): + """ + Inserts `spaces` after each string of new lines in `s` + and before the start of the string. + """ + new = re.sub('(\n+)', '\\1%s' % (' ' * spaces), s) + return (' ' * spaces) + new.strip() + + def docstring(d): + if len(d.docstring) == 0 and hasattr(d, 'inherits'): + return d.inherits.docstring + else: + return d.docstring +%> + +<%def name="function(func)" filter="trim"> +${func.name}(${func.spec()}) +${docstring(func) | indent} + + +<%def name="variable(var)" filter="trim"> +${var.name} +${docstring(var) | indent} + + +<%def name="class_(cls)" filter="trim"> +${cls.name} \ +% if len(cls.docstring) > 0: + +${cls.docstring | indent} +% endif +<% + class_vars = cls.class_variables() + static_methods = cls.functions() + inst_vars = cls.instance_variables() + methods = cls.methods() + mro = cls.module.mro(cls) + descendents = cls.module.descendents(cls) +%> +% if len(mro) > 0: + Ancestors (in MRO) + ------------------ + % for c in mro: + ${c.refname} + % endfor + +% endif +% if len(descendents) > 0: + Descendents + ----------- + % for c in descendents: + ${c.refname} + % endfor + +% endif +% if len(class_vars) > 0: + Class variables + --------------- + % for v in class_vars: +${capture(variable, v) | indent} + + % endfor +% endif +% if len(static_methods) > 0: + Static methods + -------------- + % for f in static_methods: +${capture(function, f) | indent} + + % endfor +% endif +% if len(inst_vars) > 0: + Instance variables + ------------------ + % for v in inst_vars: +${capture(variable, v) | indent} + + % endfor +% endif +% if len(methods) > 0: + Methods + ------- + % for m in methods: +${capture(function, m) | indent} + + % endfor +% endif + + +## Start the output logic for an entire module. + +<% + variables = module.variables() + classes = module.classes() + functions = module.functions() + submodules = module.submodules +%> + +Module ${module.name} +-------${'-' * len(module.name)} +${module.docstring} + + +% if len(variables) > 0: +Variables +--------- + % for v in variables: +${variable(v)} + + % endfor +% endif + + +% if len(functions) > 0: +Functions +--------- + % for f in functions: +${function(f)} + + % endfor +% endif + + +% if len(classes) > 0: +Classes +------- + % for c in classes: +${class_(c)} + + % endfor +% endif + + +% if len(submodules) > 0: +Sub-modules +----------- + % for m in submodules: + ${m.name} + % endfor +% endif diff --git a/pdocs/version.py b/pdocs/version.py new file mode 100644 index 0000000..48cf6a2 --- /dev/null +++ b/pdocs/version.py @@ -0,0 +1 @@ +VERSION = "0.4" diff --git a/pdocs/web.py b/pdocs/web.py new file mode 100644 index 0000000..c2519a0 --- /dev/null +++ b/pdocs/web.py @@ -0,0 +1,148 @@ +import http.server +import logging +import os.path +import re + +import pdocs.doc +import pdocs.extract +import pdocs.render + + +class DocHandler(http.server.BaseHTTPRequestHandler): + def do_HEAD(self): + if self.path != "/": + out = self.html() + if out is None: + self.send_response(404) + self.end_headers() + return + + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + + def do_GET(self): + if self.path == "/": + midx = [] + for m in self.server.modules: + midx.append((m.name, m.docstring)) + midx = sorted(midx, key=lambda x: x[0].lower()) + out = pdocs.render.html_index(midx, self.server.args.link_prefix) + elif self.path.endswith(".ext"): + # External links are a bit weird. You should view them as a giant + # hack. Basically, the idea is to "guess" where something lives + # when documenting another module and hope that guess can actually + # track something down in a more global context. + # + # The idea here is to start specific by looking for HTML that + # exists that matches the full external path given. Then trim off + # one component at the end and try again. + # + # If no HTML is found, then we ask `pdoc` to do its thang on the + # parent module in the external path. If all goes well, that + # module will then be able to find the external identifier. + + import_path = self.path[:-4].lstrip("/") + resolved = self.resolve_ext(import_path) + if resolved is None: # Try to generate the HTML... + logging.info("Generating HTML for %s on the fly..." % import_path) + self.html(import_path.split(".")[0]) + + # Try looking once more. + resolved = self.resolve_ext(import_path) + if resolved is None: # All hope is lost. + self.send_response(404) + self.send_header("Content-type", "text/html") + self.end_headers() + self.echo("External identifier %s not found." % import_path) + return + self.send_response(302) + self.send_header("Location", resolved) + self.end_headers() + return + else: + out = self.html() + if out is None: + self.send_response(404) + self.send_header("Content-type", "text/html") + self.end_headers() + + err = "Module %s not found." % self.import_path + self.echo(err) + return + + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + self.echo(out) + + def echo(self, s): + self.wfile.write(s.encode("utf-8")) + + def html(self): + """ + Retrieves and sends the HTML belonging to the path given in + URL. This method is smart and will look for HTML files already + generated and account for whether they are stale compared to + the source code. + """ + # Deny favico shortcut early. + if self.path == "/favicon.ico": + return None + return pdocs.render.html_module(pdocs.extract.extract_module(self.import_path)) + + def resolve_ext(self, import_path): + def exists(p): + p = os.path.join(self.server.args.html_dir, p) + pkg = os.path.join(p, pdocs.render.html_package_name) + mod = p + pdocs.render.html_module_suffix + + if os.path.isfile(pkg): + return pkg[len(self.server.args.html_dir) :] + elif os.path.isfile(mod): + return mod[len(self.server.args.html_dir) :] + return None + + parts = import_path.split(".") + for i in range(len(parts), 0, -1): + p = os.path.join(*parts[0:i]) + realp = exists(p) + if realp is not None: + return "/%s#%s" % (realp.lstrip("/"), import_path) + return None + + @property + def file_path(self): + fp = os.path.join(self.server.args.html_dir, *self.import_path.split(".")) + pkgp = os.path.join(fp, pdocs.render.html_package_name) + if os.path.isdir(fp) and os.path.isfile(pkgp): + fp = pkgp + else: + fp += pdocs.render.html_module_suffix + return fp + + @property + def import_path(self): + pieces = self.clean_path.split("/") + if pieces[-1].startswith(pdocs.render.html_package_name): + pieces = pieces[:-1] + if pieces[-1].endswith(pdocs.render.html_module_suffix): + pieces[-1] = pieces[-1][: -len(pdocs.render.html_module_suffix)] + return ".".join(pieces) + + @property + def clean_path(self): + new, _ = re.subn("//+", "/", self.path) + if "#" in new: + new = new[0 : new.index("#")] + return new.strip("/") + + def address_string(self): + return "%s:%s" % (self.client_address[0], self.client_address[1]) + + +class DocServer(http.server.HTTPServer): + def __init__(self, addr, args, modules): + self.args = args + self.modules = modules + super().__init__(addr, DocHandler) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..64508cf --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,30 @@ +[tool.poetry] +name = "pdocs" +version = "0.1.0" +description = "A simple program and library to auto generate API documentation for Python modules." +authors = ["Timothy Crosley "] +license = "MIT" +readme = "README.md" + +[tool.poetry.dependencies] +python = "^3.6" +Markdown = "^3.0.0" +Mako = "^1.1" +hug = "^2.6" + +[tool.poetry.dev-dependencies] +mypy = "^0.720.0" +isort = "^4.3" +pytest = "^5.1" +pytest-cov = "^2.7" +flake8-bugbear = "^19.8" +bandit = "^1.6" +vulture = "^1.0" +safety = "^1.8" +black = {version = "^18.3-alpha.0", allows-prereleases = true} +pytest-mock = "^1.10" +ipython = "^7.7" + +[build-system] +requires = ["poetry>=0.12"] +build-backend = "poetry.masonry.api" diff --git a/scripts/clean.sh b/scripts/clean.sh new file mode 100755 index 0000000..546179e --- /dev/null +++ b/scripts/clean.sh @@ -0,0 +1,4 @@ +#!/bin/bash -xe + +poetry run isort --recursive pdocs tests/ +poetry run black pdocs/ tests/ -l 100 --exclude malformed diff --git a/scripts/lint.sh b/scripts/lint.sh new file mode 100755 index 0000000..62351dc --- /dev/null +++ b/scripts/lint.sh @@ -0,0 +1,8 @@ +#!/bin/bash -xe + +poetry run mypy --ignore-missing-imports pdocs/ +poetry run isort --check --diff --recursive pdocs/ tests/ +poetry run black --check -l 100 pdocs/ tests/ --exclude malformed +poetry run flake8 --max-line 100 --ignore F403,F401,W503 --exclude mitmproxy/contrib/*,test/mitmproxy/data/*,release/build/*,*malformed* +poetry run safety check +poetry run bandit -r pdocs diff --git a/scripts/test.sh b/scripts/test.sh new file mode 100755 index 0000000..c272a56 --- /dev/null +++ b/scripts/test.sh @@ -0,0 +1,4 @@ +#!/bin/bash -xe + +# ./scripts/lint.sh +poetry run pytest --cov=pdocs --cov=tests --cov-report=term-missing ${@} --cov-report html tests --capture=no --color=yes diff --git a/tests/modules/README b/tests/modules/README new file mode 100644 index 0000000..ed552a2 --- /dev/null +++ b/tests/modules/README @@ -0,0 +1 @@ +Test modules that are not on the Python path diff --git a/tests/modules/dirmod/__init__.py b/tests/modules/dirmod/__init__.py new file mode 100644 index 0000000..20bbd7b --- /dev/null +++ b/tests/modules/dirmod/__init__.py @@ -0,0 +1,5 @@ +def simple(): + """ + A docstring. + """ + pass diff --git a/tests/modules/index/__init__.py b/tests/modules/index/__init__.py new file mode 100644 index 0000000..3d98b33 --- /dev/null +++ b/tests/modules/index/__init__.py @@ -0,0 +1 @@ +ROOT = 1 diff --git a/tests/modules/index/index.py b/tests/modules/index/index.py new file mode 100644 index 0000000..07a5a99 --- /dev/null +++ b/tests/modules/index/index.py @@ -0,0 +1 @@ +INDEX = 1 diff --git a/tests/modules/index/two/__init__.py b/tests/modules/index/two/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/modules/malformed/syntax.py b/tests/modules/malformed/syntax.py new file mode 100644 index 0000000..7510ad8 --- /dev/null +++ b/tests/modules/malformed/syntax.py @@ -0,0 +1,4 @@ +# fmt: off +# Syntax error +class +# fmt: on diff --git a/tests/modules/one.py b/tests/modules/one.py new file mode 100644 index 0000000..20bbd7b --- /dev/null +++ b/tests/modules/one.py @@ -0,0 +1,5 @@ +def simple(): + """ + A docstring. + """ + pass diff --git a/tests/modules/submods/__init__.py b/tests/modules/submods/__init__.py new file mode 100644 index 0000000..20bbd7b --- /dev/null +++ b/tests/modules/submods/__init__.py @@ -0,0 +1,5 @@ +def simple(): + """ + A docstring. + """ + pass diff --git a/tests/modules/submods/three/__init__.py b/tests/modules/submods/three/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/modules/submods/two.py b/tests/modules/submods/two.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/onpath/README b/tests/onpath/README new file mode 100644 index 0000000..8894e79 --- /dev/null +++ b/tests/onpath/README @@ -0,0 +1 @@ +Test modules that are on the Python path from the test perspective. diff --git a/tests/onpath/__init__.py b/tests/onpath/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/onpath/malformed_syntax.py b/tests/onpath/malformed_syntax.py new file mode 100644 index 0000000..8742252 --- /dev/null +++ b/tests/onpath/malformed_syntax.py @@ -0,0 +1,3 @@ + +# Syntax error +class diff --git a/tests/onpath/simple.py b/tests/onpath/simple.py new file mode 100644 index 0000000..a179db9 --- /dev/null +++ b/tests/onpath/simple.py @@ -0,0 +1,2 @@ +def simple(): + pass diff --git a/tests/test_doc.py b/tests/test_doc.py new file mode 100644 index 0000000..2864bdd --- /dev/null +++ b/tests/test_doc.py @@ -0,0 +1,37 @@ +import pdocs.doc +import pdocs.extract +import tutils + + +def test_simple(): + with tutils.tdir(): + m = pdocs.extract.extract_module("./modules/one.py") + assert m + + +class Dummy: + def method(self): + pass + + @classmethod + def class_method(cls): + pass + + @staticmethod + def static_method(): + pass + + +class DummyChild(Dummy): + def class_method(self): + pass + + +def test_is_static(): + assert pdocs.doc._is_method(Dummy, "method") + assert not pdocs.doc._is_method(Dummy, "class_method") + assert not pdocs.doc._is_method(Dummy, "static_method") + + assert pdocs.doc._is_method(DummyChild, "method") + assert pdocs.doc._is_method(DummyChild, "class_method") + assert not pdocs.doc._is_method(Dummy, "static_method") diff --git a/tests/test_extract.py b/tests/test_extract.py new file mode 100644 index 0000000..8fe50d3 --- /dev/null +++ b/tests/test_extract.py @@ -0,0 +1,93 @@ +import pytest + +import pdocs.extract +import tutils + + +@pytest.mark.parametrize( + "input,expected", + [ + ("foo", ("", "foo")), + ("foo.bar", ("", "foo.bar")), + ("foo/bar.py", ("foo", "bar")), + ("./bar.py", (".", "bar")), + ("./bar.foo", None), + ("", None), + ], +) +def test_split_module_spec(input, expected): + if expected is None: + with pytest.raises(pdocs.extract.ExtractError): + pdocs.extract.split_module_spec(input) + else: + assert pdocs.extract.split_module_spec(input) == expected + + +@pytest.mark.parametrize( + "path,mod,expected,match", + [ + ("./modules", "one", False, None), + ("./modules", "dirmod", True, None), + ("", "email", True, None), + ("", "csv", False, None), + ("", "html.parser", False, None), + ("", "onpath.simple", False, None), + ("./modules", "nonexistent", False, "not found"), + ("./modules/nonexistent", "foo", False, "not found"), + ("", "nonexistent.module", False, "not found"), + ("./modules/malformed", "syntax", False, "Error importing"), + ("", "onpath.malformed_syntax", False, "Error importing"), + ], +) +def test_load_module(path, mod, expected, match): + with tutils.tdir(): + if match: + with pytest.raises(pdocs.extract.ExtractError, match=match): + pdocs.extract.load_module(path, mod) + else: + _, ispkg = pdocs.extract.load_module(path, mod) + assert ispkg == expected + + +@pytest.mark.parametrize( + "path,expected,match", + [ + ("./modules/nonexistent.py", None, "not found"), + ("./modules/nonexistent/foo", None, "not found"), + ("nonexistent", None, "not found"), + ("nonexistent.module", None, "not found"), + ("./modules/one.two", None, "Invalid module name"), + ("./modules/malformed/syntax.py", None, "Error importing"), + ("onpath.malformed_syntax", None, "Error importing"), + ("./modules/one.py", ["one"], None), + ("./modules/one", ["one"], None), + ("./modules/dirmod", ["dirmod"], None), + ("./modules/submods", ["submods", "submods.three", "submods.two"], None), + ("csv", ["csv"], None), + ("html.parser", ["html.parser"], None), + ("onpath.simple", ["onpath.simple"], None), + ], +) +def test_extract_module(path, expected, match): + with tutils.tdir(): + if match: + with pytest.raises(pdocs.extract.ExtractError, match=match): + pdocs.extract.extract_module(path) + else: + ret = pdocs.extract.extract_module(path) + assert sorted([i.name for i in ret.allmodules()]) == expected + + +@pytest.mark.parametrize( + "path,modname,expected", + [ + ("./modules", "one", []), + ("./modules", "dirmod", []), + ("./modules", "submods", ["submods.three", "submods.two"]), + ("./modules", "malformed", ["malformed.syntax"]), + ], +) +def test_submodules(path, modname, expected): + with tutils.tdir(): + ret = pdocs.extract.submodules(path, modname) + assert ret == expected diff --git a/tests/test_pdoc.py b/tests/test_pdoc.py new file mode 100644 index 0000000..b862d25 --- /dev/null +++ b/tests/test_pdoc.py @@ -0,0 +1,5 @@ +import pdocs + + +def test_pdocs(): + assert pdocs diff --git a/tests/test_render.py b/tests/test_render.py new file mode 100644 index 0000000..60fb2c7 --- /dev/null +++ b/tests/test_render.py @@ -0,0 +1,19 @@ +import pdocs.doc +import pdocs.extract +import pdocs.render +import tutils + + +def test_html_module(): + with tutils.tdir(): + m = pdocs.extract.extract_module("./modules/one") + assert pdocs.render.html_module(m) + + +def test_html_module_index(): + with tutils.tdir(): + roots = [ + pdocs.extract.extract_module("./modules/one"), + pdocs.extract.extract_module("./modules/submods"), + ] + assert pdocs.render.html_index(roots) diff --git a/tests/test_static.py b/tests/test_static.py new file mode 100644 index 0000000..56ca45b --- /dev/null +++ b/tests/test_static.py @@ -0,0 +1,56 @@ +import pathlib + +import pytest + +import pdocs.extract +import pdocs.static +import tutils + + +@pytest.mark.parametrize( + "modspec,ident,path", + [ + ("./modules/one", "one", "one.html"), + ("./modules/dirmod", "dirmod", "dirmod.html"), + ("./modules/submods", "submods", "submods/index.html"), + ("./modules/submods", "submods.two", "submods/two.html"), + ("./modules/submods", "submods.three", "submods/three.html"), + ("./modules/index", "index", "index/index.html"), + ("./modules/index", "index.index", "index/index.m.html"), + ], +) +def test_module_path(modspec, ident, path): + with tutils.tdir(): + root = pdocs.extract.extract_module(modspec) + submod = root.find_ident(ident) + + mp = pdocs.static.module_to_path(submod) + assert mp == pathlib.Path(path) + + retmod = pdocs.static.path_to_module([root], mp) + assert retmod.name == submod.name + + retmod = pdocs.static.path_to_module([root], mp.with_suffix("")) + assert retmod.name == submod.name + + +def test_path_to_module(): + with tutils.tdir(): + root = pdocs.extract.extract_module("./modules/submods") + with pytest.raises(pdocs.static.StaticError): + pdocs.static.path_to_module([root], pathlib.Path("nonexistent")) + + +def test_static(tmpdir): + dst = pathlib.Path(str(tmpdir)) + with tutils.tdir(): + one = pdocs.extract.extract_module("./modules/one") + two = pdocs.extract.extract_module("./modules/submods") + assert not pdocs.static.would_overwrite(dst, [one]) + assert not pdocs.static.would_overwrite(dst, [one, two]) + pdocs.static.html_out(dst, [one]) + assert pdocs.static.would_overwrite(dst, [one]) + assert pdocs.static.would_overwrite(dst, [one, two]) + pdocs.static.html_out(dst, [one, two]) + assert pdocs.static.would_overwrite(dst, [one]) + assert pdocs.static.would_overwrite(dst, [one, two]) diff --git a/tests/tutils.py b/tests/tutils.py new file mode 100644 index 0000000..f0ae34b --- /dev/null +++ b/tests/tutils.py @@ -0,0 +1,13 @@ +import contextlib +import os + + +@contextlib.contextmanager +def tdir(): + """ + A small helper to place us within the test directory. + """ + old_dir = os.getcwd() + os.chdir(os.path.dirname(__file__)) + yield + os.chdir(old_dir) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..f9d0e33 --- /dev/null +++ b/tox.ini @@ -0,0 +1,21 @@ +[tox] +envlist = py37, lint +skipsdist = True +toxworkdir={env:TOX_WORK_DIR:.tox} + +[testenv] +deps = + {env:CI_DEPS:} + -rrequirements.txt +passenv = CODECOV_TOKEN CI CI_* TRAVIS TRAVIS_* APPVEYOR APPVEYOR_* SNAPSHOT_* OPENSSL RTOOL_* +setenv = HOME = {envtmpdir} +commands = + pdoc --version + pytest --timeout 60 --cov-report='' --cov=pdoc {posargs} + {env:CI_COMMANDS:python -c ""} + +[testenv:lint] +commands = + pdoc --version + flake8 pdoc test + mypy --ignore-missing-imports ./pdoc