Skip to content

Commit

Permalink
Ensure API reference documentation works for multiple large packages (#…
Browse files Browse the repository at this point in the history 
…65)

Fix documentation syntax typo.

New tests for nested module packages.
Ensure correct mkdocstrings paths for multi-packages.

Add logging to tasks.
  • Loading branch information
@CasperWA
CasperWA authored Oct 6, 2022
1 parent e35d850 commit 0d34c29
Show file tree
Hide file tree
Showing 3 changed files with 247 additions and 10 deletions.
77 changes: 71 additions & 6 deletions ci_cd/tasks.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""Repository management tasks powered by `invoke`.
More information on `invoke` can be found at [pyinvoke.org](http://www.pyinvoke.org/).
"""
import logging
import os
import re
import shutil
Expand All @@ -20,6 +21,10 @@
from invoke import Context, Result


LOGGER = logging.getLogger(__file__)
LOGGER.setLevel(logging.DEBUG)


class Emoji(str, Enum):
"""Unicode strings for certain emojis."""

Expand Down Expand Up @@ -232,8 +237,10 @@ def setver( # pylint: disable=too-many-locals
code_base_update_separator
)
except ValueError:
msg = traceback.format_exc()
LOGGER.error(msg)
if test:
print(traceback.format_exc())
print(msg)
sys.exit(
f"{Emoji.CROSS_MARK.value} Error: Could not properly extract "
"'file path', 'pattern', 'replacement string' from the "
Expand All @@ -255,10 +262,24 @@ def setver( # pylint: disable=too-many-locals
errors.append(error_msg)
continue

LOGGER.debug(
"""filepath: %s
pattern: %r
replacement (input): %s
replacement (handled): %s
""",
filepath,
pattern,
replacement,
replacement.format(
**{"package_dir": package_dir, "version": semantic_version}
),
)
if test:
print(f"filepath: {filepath}")
print(f"pattern: {pattern!r}")
print(f"replacement (input): {replacement}")
print(
f"filepath: {filepath}\npattern: {pattern!r}\n"
f"replacement (input): {replacement}"
)
print(
"replacement (handled): "
f"{replacement.format(**{'package_dir': package_dir, 'version': semantic_version})}" # pylint: disable=line-too-long
Expand All @@ -275,8 +296,10 @@ def setver( # pylint: disable=too-many-locals
),
)
except re.error:
msg = traceback.format_exc()
LOGGER.error(msg)
if test:
print(traceback.format_exc())
print(msg)
sys.exit(
f"{Emoji.CROSS_MARK.value} Error: Could not update file {filepath}"
f" according to the given input:\n\n pattern: {pattern}\n "
Expand Down Expand Up @@ -349,6 +372,7 @@ def update_deps( # pylint: disable=too-many-branches,too-many-locals,too-many-s
)
if match is None:
msg = f"Could not parse package, operator, and version for line:\n {line}"
LOGGER.warning(msg)
if fail_fast:
sys.exit(msg)
print(msg)
Expand Down Expand Up @@ -377,6 +401,7 @@ def update_deps( # pylint: disable=too-many-branches,too-many-locals,too-many-s
"Could not parse package and version from 'pip index versions' output "
f"for line:\n {package_latest_version_line}"
)
LOGGER.warning(msg)
if fail_fast:
sys.exit(msg)
print(msg)
Expand All @@ -388,6 +413,7 @@ def update_deps( # pylint: disable=too-many-branches,too-many-locals,too-many-s
" the name returned from 'pip index versions': "
f"{match.group('package')!r}"
)
LOGGER.warning(msg)
if fail_fast:
sys.exit(msg)
print(msg)
Expand Down Expand Up @@ -554,6 +580,22 @@ def write_file(full_path: Path, content: str) -> None:
package_dirs: list[Path] = [root_repo_path / _ for _ in package_dir]
docs_api_ref_dir = root_repo_path / docs_folder / "api_reference"

LOGGER.debug(
"""package_dirs: %s
docs_api_ref_dir: %s
unwanted_folder: %s
unwanted_file: %s
full_docs_folder: %s
full_docs_file: %s
special_option: %s""",
package_dirs,
docs_api_ref_dir,
unwanted_folder,
unwanted_file,
full_docs_folder,
full_docs_file,
special_option,
)
if debug:
print("package_dirs:", package_dirs, flush=True)
print("docs_api_ref_dir:", docs_api_ref_dir, flush=True)
Expand All @@ -566,6 +608,9 @@ def write_file(full_path: Path, content: str) -> None:
special_options_files = defaultdict(list)
for special_file, option in [_.split(",", maxsplit=1) for _ in special_option]:
if any("," in _ for _ in (special_file, option)):
LOGGER.error(
"Failing for special-option: %s", ",".join([special_file, option])
)
if debug:
print(
"Failing for special-option:",
Expand All @@ -578,6 +623,7 @@ def write_file(full_path: Path, content: str) -> None:
)
special_options_files[special_file].append(option)

LOGGER.debug("special_options_files: %s", special_options_files)
if debug:
print("special_options_files:", special_options_files, flush=True)

Expand All @@ -594,13 +640,15 @@ def write_file(full_path: Path, content: str) -> None:
)

if docs_api_ref_dir.exists() and pre_clean:
LOGGER.debug("Removing %s", docs_api_ref_dir)
if debug:
print(f"Removing {docs_api_ref_dir}", flush=True)
shutil.rmtree(docs_api_ref_dir, ignore_errors=True)
if docs_api_ref_dir.exists():
sys.exit(f"{docs_api_ref_dir} should have been removed!")
docs_api_ref_dir.mkdir(exist_ok=True)

LOGGER.debug("Writing file: %s", docs_api_ref_dir / ".pages")
if debug:
print(f"Writing file: {docs_api_ref_dir / '.pages'}", flush=True)
write_file(
Expand All @@ -615,6 +663,7 @@ def write_file(full_path: Path, content: str) -> None:
for package in package_dirs:
for dirpath, dirnames, filenames in os.walk(package):
for unwanted in unwanted_folder:
LOGGER.debug("unwanted: %s\ndirnames: %s", unwanted, dirnames)
if debug:
print("unwanted:", unwanted, flush=True)
print("dirnames:", dirnames, flush=True)
Expand All @@ -628,21 +677,25 @@ def write_file(full_path: Path, content: str) -> None:
abspath = (
package / relpath if single_package else package.parent / relpath
).resolve()
LOGGER.debug("relpath: %s\nabspath: %s", relpath, abspath)
if debug:
print("relpath:", relpath, flush=True)
print("abspath:", abspath, flush=True)

if not (abspath / "__init__.py").exists():
# Avoid paths that are not included in the public Python API
LOGGER.debug("does not exist: %s", abspath / "__init__.py")
print("does not exist:", abspath / "__init__.py", flush=True)
continue

# Create `.pages`
docs_sub_dir = docs_api_ref_dir / relpath
docs_sub_dir.mkdir(exist_ok=True)
LOGGER.debug("docs_sub_dir: %s", docs_sub_dir)
if debug:
print("docs_sub_dir:", docs_sub_dir, flush=True)
if str(relpath) != ".":
LOGGER.debug("Writing file: %s", docs_sub_dir / ".pages")
if debug:
print(f"Writing file: {docs_sub_dir / '.pages'}", flush=True)
write_file(
Expand All @@ -666,6 +719,10 @@ def write_file(full_path: Path, content: str) -> None:
# Not a Python file: We don't care about it!
# Or filename is in the list of unwanted files:
# We don't want it!
LOGGER.debug(
"%s is not a Python file or is an unwanted file (through user input). Skipping it.",
filename,
)
if debug:
print(
f"{filename} is not a Python file or is an unwanted file "
Expand All @@ -681,8 +738,11 @@ def write_file(full_path: Path, content: str) -> None:
f"{py_path_root}/{filename.stem}".replace("/", ".")
if str(relpath) == "."
or (str(relpath) == package.name and not single_package)
else f"{py_path_root}/{relpath}/{filename.stem}".replace("/", ".")
else f"{py_path_root}/{relpath if single_package else relpath.relative_to(package.name)}/{filename.stem}".replace(
"/", "."
)
)
LOGGER.debug("filename: %s\npy_path: %s", filename, py_path)
if debug:
print("filename:", filename, flush=True)
print("py_path:", py_path, flush=True)
Expand Down Expand Up @@ -710,6 +770,11 @@ def write_file(full_path: Path, content: str) -> None:
)
template += "\n"

LOGGER.debug(
"template: %s\nWriting file: %s",
template,
docs_sub_dir / filename.with_suffix(".md"),
)
if debug:
print("template:", template, flush=True)
print(
Expand Down
4 changes: 1 addition & 3 deletions docs/hooks/docs_api_reference.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,7 @@ Any of these options can be given through the `args` key when defining the hook.
| `--unwanted-file` | A file to avoid including into the Python API reference documentation. If this is not supplied, it will default to `__init__.py`</br></br>**Note**: Only full file names, not paths, may be included, i.e., filename + file extension.</br></br>**Note**: All files with these names will be excluded.</br></br>This input option can be supplied multiple times. | No | \_\_init\_\_.py | _string_ |
| `--full-docs-folder` | A folder in which to include everything - even those without documentation strings. This may be useful for a module full of data models or to ensure all class attributes are listed.</br></br>This input option can be supplied multiple times. | No | _Empty string_ | _string_ |
| `--full-docs-file` | A full relative path to a file in which to include everything - even those without documentation strings. This may be useful for a file full of data models or to ensure all class attributes are listed.</br></br>This input option can be supplied multiple times. | No | _Empty string_ | _string_ |
| `--special-option` | A combination of a relative path to a file and a fully formed mkdocstrings option that should be added to the generated MarkDown file. The combination should be comma-separated.</br>Example:
`my_module/py_file.py,show_bases:false`.</br></br>Encapsulate the value in double quotation marks (`"`) if including spaces ( ).</br></br>**Important**: If multiple package-dir options are supplied, the relative path MUST include/start with the package-dir value, e.g., `"my_package/my_module/py_file.py,show_bases: false"`.</br></br>This input option can be supplied multiple times. The options will be accumulated
for the same file, if given several times. | No | _Empty string_ | _string_ |
| `--special-option` | A combination of a relative path to a file and a fully formed mkdocstrings option that should be added to the generated MarkDown file. The combination should be comma-separated.</br>Example: `my_module/py_file.py,show_bases:false`.</br></br>Encapsulate the value in double quotation marks (`"`) if including spaces ( ).</br></br>**Important**: If multiple package-dir options are supplied, the relative path MUST include/start with the package-dir value, e.g., `"my_package/my_module/py_file.py,show_bases: false"`.</br></br>This input option can be supplied multiple times. The options will be accumulated for the same file, if given several times. | No | _Empty string_ | _string_ |
| `--relative` | Whether or not to use relative Python import links in the API reference markdown files. See section [Using it together with CI/CD workflows](#using-it-together-with-cicd-workflows) above. | No | `False` | _boolean_ |
| `--debug` | Whether or not to print debug statements. | No | `False` | _boolean_ |

Expand Down
Loading

0 comments on commit 0d34c29

Please sign in to comment.