From 1cf36a4e28d7e42461faab8382fb0375d17fe110 Mon Sep 17 00:00:00 2001 From: Martin Barisits Date: Thu, 28 Nov 2024 10:14:46 +0100 Subject: [PATCH] Revert "Replace building of python client documentation with pydoc to mkdocstrings" This reverts commit 6114f2c7af218e7a508f3f947fa86cbae1c020b5. --- tools/build_documentation.sh | 3 +- .../run_in_docker/generate_client_api_docs.sh | 19 +- .../generate_client_api_pages.py | 24 --- tools/run_in_docker/mkdocs.yml | 51 ----- tools/run_in_docker/rucio_processor.py | 193 ++++++++++++++++++ tools/run_in_docker/rucio_renderer.py | 64 ++++++ website/docusaurus.config.js | 2 +- website/sidebars.json | 8 + 8 files changed, 283 insertions(+), 81 deletions(-) delete mode 100755 tools/run_in_docker/generate_client_api_pages.py delete mode 100755 tools/run_in_docker/mkdocs.yml create mode 100644 tools/run_in_docker/rucio_processor.py create mode 100644 tools/run_in_docker/rucio_renderer.py diff --git a/tools/build_documentation.sh b/tools/build_documentation.sh index e38dd6b844..533104cd9c 100755 --- a/tools/build_documentation.sh +++ b/tools/build_documentation.sh @@ -35,9 +35,10 @@ mkdir -p "$SCRIPT_DIR"/../website/static/html/ cp -r "$AUTO_GENERATED"/rest_api_doc_spec.yaml "$SCRIPT_DIR"/../website/static/yaml/ cp -r "$AUTO_GENERATED"/rest_api_doc.html "$SCRIPT_DIR"/../website/static/html/ -cp -r "$AUTO_GENERATED"/site "$SCRIPT_DIR"/../website/static/html/ +cp -r "$AUTO_GENERATED"/client_api "$DOCS" cp -r "$AUTO_GENERATED"/bin "$DOCS" + echo "Generating Release Notes..." "$SCRIPT_DIR"/generate_release_notes.py echo "Generating Release Notes Index..." diff --git a/tools/run_in_docker/generate_client_api_docs.sh b/tools/run_in_docker/generate_client_api_docs.sh index 802fffeafc..fe436df54f 100755 --- a/tools/run_in_docker/generate_client_api_docs.sh +++ b/tools/run_in_docker/generate_client_api_docs.sh @@ -4,11 +4,22 @@ set -e echo "Generating the Client Api Docs..." -#pip install --upgrade "pydoc-markdown>3" &> /dev/null -pip install --upgrade mkdocs mkdocs-gen-files mkdocstrings-python mkdocs-material +pip install --upgrade "pydoc-markdown>3" &> /dev/null mkdir -p /auto_generated/client_api +for f in rucio/lib/rucio/client/*.py; do + if [[ $f =~ "__init__" ]]; then + continue + fi -mkdocs build --clean --no-directory-urls + executable_name=$(basename "$f" ".py") -cp -r site /auto_generated/ \ No newline at end of file + config=" +processors: + - type: rucio_processor.RucioProcessor +renderer: + type: rucio_renderer.RucioRenderer" + content=$(PYTHONPATH=. pydoc-markdown -I rucio/lib/rucio/client -m "$executable_name" "$config") + + echo "$content" > /auto_generated/client_api/"$executable_name".md +done diff --git a/tools/run_in_docker/generate_client_api_pages.py b/tools/run_in_docker/generate_client_api_pages.py deleted file mode 100755 index b28036e5e0..0000000000 --- a/tools/run_in_docker/generate_client_api_pages.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Generate the code reference pages and navigation.""" - -from pathlib import Path -import os -import mkdocs_gen_files - -src = Path("rucio/lib/rucio/client/") -root = src.parent -doc_path = "/auto_generated/client_api" -files = os.listdir(doc_path) -for file in files: - os.remove(f"{doc_path}/{file}") - - -py_files = [f for f in list(src.rglob("*.py")) if "__init__.py" not in f.name] - -for path in py_files: - module_name = path.name.split('.py')[0] - full_doc_path = Path(f"{doc_path}/{module_name}.md") - - with mkdocs_gen_files.open(full_doc_path, "a") as fd: - module_path = path.relative_to(src).with_suffix("") - fd.write(f"::: rucio.client.{module_path}") - fd.write("\n\n") \ No newline at end of file diff --git a/tools/run_in_docker/mkdocs.yml b/tools/run_in_docker/mkdocs.yml deleted file mode 100755 index 61b7cfd164..0000000000 --- a/tools/run_in_docker/mkdocs.yml +++ /dev/null @@ -1,51 +0,0 @@ -site_name: "Rucio Python Client Documentation" -repo_url: https://github.com/rucio/rucio/ -site_url: https://rucio.cern.ch/documentation/ -copyright: "Copyright © 2024 CERN" -docs_dir: /auto_generated/client_api -use_directory_urls: true -not_in_nav: - /auto_generated - -theme: - name: material - logo: ../../img/rucio_horizontaled_black_cropped.svg - extra_css: ../../../src/customTheme.css - extra: - homepage: https://rucio.cern.ch/documentation/ - features: - - content.code.copy - - toc.follow - palette: - - scheme: default - primary: white - toggle: - icon: material/brightness-7 - name: Switch to dark mode - - scheme: slate - primary: white - toggle: - icon: material/brightness-4 - name: Switch to light mode - -markdown_extensions: -- toc: - permalink: true -plugins: -- gen-files: - scripts: - - generate_client_api_pages.py -- search -- autorefs -- mkdocstrings: - default_handler: python - handlers: - python: - options: - members_order: "source" - heading_level: 2 - show_source: false - allow_inspection: false - merge_init_into_class: true - docstring_style: "sphinx" - show_bases: false diff --git a/tools/run_in_docker/rucio_processor.py b/tools/run_in_docker/rucio_processor.py new file mode 100644 index 0000000000..1b022049ea --- /dev/null +++ b/tools/run_in_docker/rucio_processor.py @@ -0,0 +1,193 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2019 Niklas Rosenstein +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import dataclasses +import logging +import typing as t + +import docspec +import docstring_parser +from pydoc_markdown.interfaces import Processor, Resolver + +logger = logging.getLogger(__name__) + + +def sanitize(s: t.Optional[str]) -> str: + if not s: + return "" + + character_map = {r"<": r"\<", r">": r"\>", r"{": r"\{", r"}": r"\}"} + + for before, after in character_map.items(): + s = s.replace(before, after) + return s + + +@dataclasses.dataclass +class _ParamLine: + """ + Helper data class for holding details of Sphinx arguments. + """ + + name: str + docs: str + type: t.Optional[str] = None + + +def generate_sections_markdown(sections): + ret = [] + + ret.append("\n") + for key, section in sections.items(): + if section: + ret.append("\n\n") + + ret.append( + "\n\n") + + ret.append( + "\n\n") + + ret.append("\n\n") + ret.append("\n
\n" # noqa: E501 + ) + ret.append(f"**{key}**:") + ret.append("\n\n" # noqa: E501 + ) + ret.extend(section) + ret.append("\n
\n") + + return ret + + +@dataclasses.dataclass +class RucioProcessor(Processor): + _KEYWORDS = { + "Arguments": [ + "arg", + "argument", + "param", + "parameter", + "type", + ], + "Returns": [ + "return", + "returns", + "rtype", + ], + "Raises": [ + "raises", + "raise", + ], + } + + def check_docstring_format(self, docstring: str) -> bool: + return any( + f":{k}" in docstring for _, value in self._KEYWORDS.items() for k in value + ) + + def process( + self, modules: t.List[docspec.Module], resolver: t.Optional[Resolver] + ) -> None: + docspec.visit(modules, self._process) + + def _convert_raises( + self, raises: t.List[docstring_parser.common.DocstringRaises] + ) -> list: + """Convert a list of DocstringRaises from docstring_parser to markdown lines + + :return: A list of markdown formatted lines + """ + converted_lines = [] + for entry in raises: + converted_lines.append( + "`{}`: {}\n".format( + sanitize(entry.type_name), sanitize(entry.description) + ) + ) + return converted_lines + + def _convert_params( + self, params: t.List[docstring_parser.common.DocstringParam] + ) -> list: + """Convert a list of DocstringParam to markdown lines. + + :return: A list of markdown formatted lines + """ + converted = [] + for param in params: + if param.type_name is None: + converted.append( + "`{name}`: {description}\n".format( + name=sanitize(param.arg_name), + description=sanitize(param.description), + ) + ) + else: + converted.append( + "`{name}` (`{type}`): {description}\n".format( + name=sanitize(param.arg_name), + type=param.type_name, + description=sanitize(param.description), + ) + ) + return converted + + def _convert_returns( + self, returns: t.Optional[docstring_parser.common.DocstringReturns] + ) -> str: + """Convert a DocstringReturns object to a markdown string. + + :return: A markdown formatted string + """ + if not returns: + return "" + if returns.type_name: + type_data = "`{}`: ".format(returns.type_name) + else: + type_data = "" + return " " + type_data + (sanitize(returns.description) or "") + "\n" + + def _process(self, node: docspec.ApiObject) -> None: + if not node.docstring: + return + + lines = [] + components: t.Dict[str, t.List[str]] = {} + + parsed_docstring = docstring_parser.parse( + node.docstring.content, docstring_parser.DocstringStyle.REST + ) + components["Arguments"] = self._convert_params(parsed_docstring.params) + components["Raises"] = self._convert_raises(parsed_docstring.raises) + return_doc = self._convert_returns(parsed_docstring.returns) + if return_doc: + components["Returns"] = [return_doc] + + if parsed_docstring.short_description: + lines.append(sanitize(parsed_docstring.short_description)) + lines.append("") + if parsed_docstring.long_description: + lines.append(sanitize(parsed_docstring.long_description)) + lines.append("") + + lines.extend(generate_sections_markdown(components)) + node.docstring.content = "\n".join(lines) diff --git a/tools/run_in_docker/rucio_renderer.py b/tools/run_in_docker/rucio_renderer.py new file mode 100644 index 0000000000..3d370647fe --- /dev/null +++ b/tools/run_in_docker/rucio_renderer.py @@ -0,0 +1,64 @@ +import dataclasses +import typing as t + +import docspec +from pydoc_markdown.contrib.renderers.markdown import MarkdownRenderer +from pydoc_markdown.interfaces import Context, Renderer + + +def get_first_client_class( + modules: t.List[docspec.Module], +) -> t.Optional[docspec.Class]: + if not modules: + return None + + for i in modules: + if isinstance(i, docspec.Class) and getattr(i, "name", "").lower().endswith( + "client" + ): + return i + + child = get_first_client_class(getattr(i, "members", [])) + if child: + return child + + return None + + +def sanitize(s: str) -> str: + character_map = {r"_": r"\_", r"<": r"\<", r">": r"\>", r"{": r"\{", r"}": r"\}"} + + for before, after in character_map.items(): + s = s.replace(before, after) + return s + + +@dataclasses.dataclass +class RucioRenderer(Renderer): + markdown: MarkdownRenderer = dataclasses.field(default_factory=MarkdownRenderer) + + def init(self, context: Context) -> None: + self.markdown.init(context) + + def render_recursive(self, obj: docspec.ApiObject) -> None: + if isinstance(obj, docspec.Function) and ( + not obj.name.startswith("_") or obj.name == "__init__" + ): + print(f"## {sanitize(obj.name)}\n") + if obj.docstring: + print('\n') + print(obj.docstring.content + "\n") + print("\n") + + for item in getattr(obj, "members", []): + self.render_recursive(item) + + def render(self, modules: t.List[docspec.Module]) -> None: + client_class = get_first_client_class(modules) + assert client_class, "Client Class should not be empty" + + print("---") + print(f"title: {client_class.name}") + print("---") + + self.render_recursive(client_class) diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 4e5e2c34cd..8dda8f2ee3 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -54,7 +54,7 @@ module.exports={ }, "items": [ { - "to": "pathname:///html/site/accountclient.html", + "to": "client_api/accountclient", "label": "Python Client API", "position": "left" }, diff --git a/website/sidebars.json b/website/sidebars.json index 069a720cd0..68b33c8ad7 100644 --- a/website/sidebars.json +++ b/website/sidebars.json @@ -32,6 +32,14 @@ "user/configuring_the_client", "user/using_the_client", "user/using_the_admin_client", + { + "Python Client API": [ + { + "type": "autogenerated", + "dirName": "client_api" + } + ] + }, "user/developing_with_rucio" ], "Operator": [