merging bib-generating logic with the docs-generating script, CSS/HTM…

…L tunings

.github/workflows/pdoc.yml

@@ -31,7 +31,7 @@ jobs:
           pip install -e .
           pip install -e examples
 
-          python -We .github/workflows/docs.py . .
+          python -We docs/generate_html.py . .
         
       - if: ${{ github.ref == 'refs/heads/main' && matrix.platform == 'ubuntu-latest' }}
         uses: JamesIves/[email protected]

docs/generate_html.py

@@ -0,0 +1,138 @@
+"""
+Script to generate docs based on given input directory (first argument)
+and output directory (second argument), includes bibliography logic
+"""
+
+import json
+import os
+import re
+import subprocess
+import sys
+
+import nbformat
+
+# <TEMPORARILY COPIED FROM DEVOPS_TESTS>
+from git.cmd import Git
+
+
+def find_files(path_to_folder_from_project_root=".", file_extension=None):
+    """
+    Returns all files in a current git repo.
+    The list of returned files may be filtered with `file_extension` param.
+    """
+    all_files = [
+        path
+        for path in Git(
+            Git(path_to_folder_from_project_root).rev_parse("--show-toplevel")
+        )
+        .ls_files()
+        .split("\n")
+        if os.path.isfile(path)
+    ]
+    if file_extension is not None:
+        return list(filter(lambda path: path.endswith(file_extension), all_files))
+
+    return all_files
+
+
+# </TEMPORARILY COPIED FROM DEVOPS_TESTS>
+
+
+def generate_badges_md_files():
+    """extracts badge-containing cell from each notebook and writes it to an .md file"""
+    for notebook_path in find_files("examples/PySDM_examples", ".ipynb"):
+        with open(notebook_path, encoding="utf8") as fin:
+            with open(notebook_path + ".badges.md", "w", encoding="utf8") as fout:
+                fout.write(nbformat.read(fin, nbformat.NO_CONVERT).cells[0].source)
+
+
+def read_dois_from_json(code_path):
+    """loads bibliography data from .json file"""
+    bibliography_json_path = f"{code_path}/docs/bibliography.json"
+    dois_from_json = {}
+    if os.path.exists(bibliography_json_path):
+        with open(bibliography_json_path, "r", encoding="utf8") as fin:
+            dois_from_json = json.load(fin)
+    return dois_from_json
+
+
+def check_dois(dois_from_json):
+    """checks if all bib entries are referenced from code and vice versa"""
+    found_dois = []
+    for extension in (".md", ".ipynb", ".py"):
+        for full_path in find_files(file_extension=extension):
+            with open(full_path, "r", encoding="utf-8") as fin:
+                text = fin.read()
+            dois = re.findall(
+                r"\b(https://doi\.org/10[.][0-9]{4,}(?:[.][0-9]+)*/(?:(?![\"&\'<>^\\])\S)+)\b",
+                text,
+            )
+            if dois:
+                found_dois.extend((full_path, doi) for doi in dois)
+
+    unique_dois_found = {doi for _, doi in found_dois}
+    unique_dois_read = set(dois_from_json.keys())
+
+    for doi in unique_dois_found:
+        assert doi in unique_dois_read, f"{doi} not found in the json file"
+    for doi in unique_dois_read:
+        assert doi in unique_dois_found, f"{doi} not referenced in the code"
+
+
+def create_references_html(dois_from_json, code_path):
+    """writes HTML file with the reference list"""
+    with open(
+        f"{code_path}/docs/templates/bibliography.html", "w", encoding="utf8"
+    ) as fout:
+        fout.write("<ol>\n")
+        for doi, data in sorted(
+            dois_from_json.items(), key=lambda item: item[1]["label"].lower()
+        ):
+            fout.write("<li>\n")
+            fout.write(
+                f'<a href="{doi}">{data["label"]}: "<em>{data["title"]}</em>"</a>\n'
+            )
+            fout.write('<ul style="list-style-type:square;font-size:smaller;">')
+            for path in data["usages"]:
+                fout.write(
+                    f'<li><a href="https://github.com/open-atmos/PySDM/tree/main/{path}">'
+                )
+                fout.write(f"{path}</a></li>")
+            fout.write("</ul>\n")
+        fout.write("</ol>\n")
+
+
+def run_pdoc(code_path, out_path):
+    """generates docs in HTML formay by executing pdoc"""
+    subprocess.run(
+        [
+            sys.executable,
+            "-We",
+            "-m",
+            "pdoc",
+            "-o",
+            f"{out_path}/html",
+            f"{code_path}/PySDM",
+            f"{code_path}/examples/PySDM_examples",
+            "-t",
+            f"{code_path}/docs/templates",
+            "--math",
+            "--mermaid",
+        ],
+        env={**os.environ, "PDOC_ALLOW_EXEC": "1"},
+        check=True,
+    )
+
+
+def _main():
+    assert len(sys.argv) == 3, f"usage: {sys.argv[0]} code_path out_path"
+    code_path, out_path = sys.argv[1], sys.argv[2]
+    generate_badges_md_files()
+    dois_from_json = read_dois_from_json(code_path)
+    check_dois(dois_from_json)
+    create_references_html(dois_from_json, code_path)
+    run_pdoc(code_path, out_path)
+
+
+if __name__ == "__main__":
+    _main()

docs/templates/custom.css

@@ -19,3 +19,7 @@ td, th {
     display: table;
     width: 100%;
 }
+
+ul {
+    margin-left: 2em;
+}

docs/templates/index.html.jinja2

@@ -166,21 +166,24 @@ jupyter-notebook PySDM/examples</code></pre>
         </p>
         <p>
           Development of PySDM was supported by:
-          <ul style="list-style:square; margin-left:1em;">
+        </p>
+        <ul style="list-style:square;">
             <li><a href="https://www.fnp.org.pl/en/">Foundation for Polish Science</a> (grant no. POIR.04.04.00-00-5E1C/18 co-financed by the European Union under the European Regional Development Fund)</li>
             <li><a href="https://ncn.gov.pl/en">Polish National Science Centre</a> (grant no. 2020/39/D/ST10/01220)</li>
             <li><a href="https://science.energy.gov">US Department of Energy</a> via the <a href="https://www.krellinst.org/csgf/">Computational Science Graduate Fellowship</a> and via the <a href="https://asr.science.energy.gov/">Atmospheric System Research Program</a> (grant no. DE-SC0021034)</li>
             <li><a href="https://www.schmidtfutures.org/our-people/#eric-schmidt">Eric</a> and <a href="https://www.schmidtfutures.org/our-people/#wendy-schmidt">Wendy Schmidt</a> (by recommendation of <a href="https://www.schmidtfutures.org/">Schmidt Futures</a>)</li>
             <li><a href="https://www.hsfoundation.org/">Heising-Simons Foundation</a></li>
             <li><a href="https://www.nsf.gov/">National Science Foundation</a> (grant AGS-1835860)</li>
-          </ul>
-        </p>
+        </ul>
     </div>
         <div>
-            <h2>Bibliography with cross-references</h2>
+            <h2>Bibliography with code cross-references</h2>
+            <p>
+                The list below summarises all literature references included in PySDM codebase and includes links to both the referenced papers, as well as to the referring PySDM source files.
+            </p>
             {% include "bibliography.html" %}
+            <h2>EOF</h2>
         </div>
-
     </main>
 
 {#    {% include "search.html.jinja2" %}#}